{
 "G_med": 2.573387642438244,
 "G_med_api_basis_P_vs_A": -1.1922125858968342,
 "G_med_flag_gt10": false,
 "addendum": {
  "cost_basis": {
   "n_items": 1489,
   "rule": "fixed-common-cost-basis-d1027 (frozen at v1.7.16)",
   "set_sha256": "4e912731e63369e1b3066b0b4afa87b44f1094aea079c5b41dc6d02e7f235537"
  },
  "link_fixed_keys": [],
  "method_change": true,
  "new_keys": [],
  "predecessor": "v1.7.23",
  "remeasured_keys": [],
  "repriced_keys": [
   "ajev-gemma4-31b-lm6q",
   "aleria-judge-1.1.1",
   "amazon-strands-decider-26b-a4b",
   "aplomb-1",
   "apus-openjev-v1-9b",
   "arcbright-elf-1-12b",
   "arcbright-elf-1-12b-r2",
   "arcbright-elf-1-31b",
   "arsenelupin-v1.1",
   "autojev-27b",
   "basal-omni-1.0-ultra",
   "bespoke-nimble-9b-v3",
   "bev-bonsai-27b",
   "blink-4b",
   "blink-v0-3-26b-a4b-nvfp4",
   "bobcat-1.3",
   "bobcat-flash-1.2",
   "chaoliangunsw-jev-style-2b-decision-v4",
   "chon-1.0-12b",
   "chon-1.0-12b-fast",
   "chon-1.0-e4b",
   "classone-gemma4-e2b",
   "classone-gemma4-e4b",
   "classone-qwen3.5-2b",
   "classone-qwen3.5-4b",
   "classone-qwen3.5-9b",
   "clef",
   "clef-flash",
   "clef-omni",
   "clm-8b",
   "coco-decision-4b",
   "compass-0.2.0",
   "cygnet",
   "d1a-gemma4-e2b",
   "d1a-gemma4-e4b",
   "decider-12b",
   "decider-12b-v1",
   "decider-2b",
   "decider-35b-a3b",
   "decider-4b-v2",
   "decider-chat-gemma4-31b",
   "decisio-gemma-4-12b-tuned-commit",
   "decisio-gemma-4-12b-tuned-commit-think",
   "decisio-gemma-4-12b-v080",
   "decisio-gemma-4-12b-v090",
   "decisio-gemma-4-31b-v080",
   "decisio-qwen3.6-35b-a3b-v090",
   "decision-2-0-vega-27b",
   "decision-4b-v11",
   "decision-4b-v12",
   "decision-fast",
   "decision2-lux-9b",
   "decisionnode-edge-0.9.2",
   "decisionnode-edge-mini-0.9.2",
   "decisor_4b",
   "deck-4b-v1-0",
   "deck31b",
   "deem-0.8-v1",
   "diffusion-jev",
   "dopest-1.0-31b",
   "eikos-27b",
   "elephant-1",
   "evalengine-decision-4b",
   "fastino-glide-1.2-open",
   "gev-26b-decide",
   "goddragon-arbiter-4b",
   "gutsy-0.8b-v0.3",
   "h2o-lightning-31b",
   "h2o-lightning-4b",
   "h2o-lightning-4b-v1.4.1",
   "hopper",
   "hopper-12b-trained",
   "imajev-2b",
   "imajev-4b-rtx5090-a2",
   "imajev-9b",
   "iota-decide-4b",
   "iota-decide-4b-v0.3",
   "jade",
   "janus-4b",
   "jeb-35b-a3b",
   "jebadiah-27b",
   "jebadiah-9b-v2",
   "jebadiah-9b-v2-1",
   "jeetu-decider-2b",
   "jeff_1_0_large",
   "jeff_1_1_large",
   "jeff_1_1_medium",
   "jeff_1_1_qwen_4b",
   "jeffan-31b-1710",
   "jeffan-4b-2010",
   "jev-27b-autotrust",
   "jev-local",
   "jev-omni",
   "jevk5-v0.3-4b",
   "jevk5-v02",
   "jevone",
   "jez-large-1.0.0",
   "jez-small-1.0.0",
   "jiwo-0.8b",
   "jiwo-4b",
   "jpt-35b-a3b",
   "jpt-4b",
   "jqv",
   "kahn1",
   "kev-0.5b",
   "kev-0.6b",
   "kev-27b",
   "kev-4b",
   "kev-8b",
   "knowledgator_gliclass_v4_9b",
   "knowline-26b-a4b-gen2.5",
   "knowline-31b-gen1",
   "knowline-35b-a3b-gen1.5",
   "knowline-4b-gen5.5",
   "krino-2b",
   "lev",
   "litjev",
   "localjev-qwen3.5-4b",
   "lumma-fev-27b",
   "lychee-27b",
   "malkuth-2b",
   "malkuth-4b",
   "mammoth-1",
   "manchego-v21-v15",
   "messier-one-v0.2",
   "messier-one-v0.4",
   "metask-jev-4b",
   "metask-jev-rain-4b",
   "metask_jev_rain_12b",
   "metask_jev_rain_12b_trained",
   "metask_jev_rain_31b_trained",
   "metask_jev_rain_31b_trained_v3",
   "mghafiri-qwen3.5-0.8b-decision-model",
   "nagi-enormous",
   "nev-2b",
   "nimble-9b",
   "nimble-v4-max-open",
   "ninfer-qwen3.8-27b",
   "ninfer-qwen3.8-flash-next",
   "noma",
   "open-jev-json-canvas-joshuasp",
   "open-jev-zefan-27b-v1.1",
   "open-jev-zefan-2b",
   "open-jev-zefan-9b",
   "openjev-thinking",
   "opensourcejev-qwen35-4b-q4km",
   "phillylabs_philly_decider_v2",
   "plumb-4b",
   "pplx-decider-flash-v1.2",
   "pplx-decider-v1-1-27b",
   "pplx-decider-v1.2",
   "purplemist-mini-1.0",
   "quyet-1-0-large",
   "quyet-1-0-medium",
   "qwen3-reranker-4b",
   "qwen35-9b-jev-data-mix-v2",
   "raw-qwen3-0.6b",
   "raw-qwen3-1.7b",
   "raw-qwen3-4b-instruct-2507",
   "raw-qwen3-8b",
   "reflex-4b",
   "reflex-instinct-0.6b",
   "rene-1-31b-fp8",
   "respan-span-02",
   "roudone-decider-4b",
   "rsi-jev-v6.1-vl-27b",
   "ryotide-qwen",
   "ryotide_qwen9",
   "seb-9b",
   "seb-9b-v2",
   "semif-qwen3.5-4b",
   "shisa-de-1",
   "sieve-31b-v2",
   "simplejev-qwen3.5-0.8b",
   "simplejev-qwen3.8-27b-selfhost",
   "spark-s1-4b-v6",
   "spx-cd-omni",
   "startlux-decision-v1.1-27b-autothink",
   "startlux-decision-v1.1-27b-instant",
   "surogate-rune-26b-a4b-v3-v16",
   "swanone",
   "system-one-sg",
   "tern-35b-a3b",
   "torchcast-decision-27b",
   "torchcast-decision-31b-v1.0",
   "torchcast-decision-4b-v1.1",
   "torchcast-decision-4b-v1.2",
   "torchcast-decision-4b-v1.3",
   "turion-jev-31b",
   "typecastlm",
   "typecastlm-1.4.0",
   "umpire-35b-a3b",
   "uniphore-4b-decider",
   "vega-u-qwen3.5-0.8b",
   "vega-u-qwen3.5-4b",
   "vegaml-0.8b",
   "vegaml-4b",
   "vera-large",
   "verdin-4b",
   "visionjev-1.0.0-27b",
   "vllm-sr-d3",
   "vllm-sr-d3-mini",
   "vllm-sr-d3-turbo",
   "wald-26b-a4b",
   "wald-4b-v2",
   "wald-4b-v2-1",
   "winnow-12b",
   "xevn-4b",
   "xor-26b-a4b",
   "zerank-2"
  ],
  "same_draw": "v1.6-regular-20261027",
  "text_fixed_keys": [
   "decisio-qwen3.6-35b-a3b-v090",
   "jeffan-31b-1710",
   "knowline-26b-a4b-gen2.5",
   "knowline-31b-gen1",
   "knowline-35b-a3b-gen1.5",
   "lychee-27b",
   "vera-large"
  ]
 },
 "additions": [
  {
   "baseline_categories_sha256": "55c86278fb2ca0e8407f58eeb5338406f298a1bca426efd2ae7b52dad2752f6f",
   "baseline_systems_sha256": "6255c1c1014f148ad59eb43f78ded416d3dbb6604ebd6af3d530935edabc8f6b",
   "date": "2026-10-07",
   "keys": [
    "kev-27b",
    "pplx-decider-v1-1-27b",
    "decider-chat-gemma4-31b",
    "jpt-35b-a3b",
    "jebadiah-27b",
    "simplejev-qwen3.8-27b-selfhost",
    "bespoke-nimble-9b-v3",
    "gev-26b-decide",
    "torchcast-decision-27b",
    "decision-2-0-vega-27b",
    "jade",
    "jev-27b-autotrust"
   ],
   "label": "a6",
   "note": "Twelve open-weights systems from the top 20 of the community Jev Decision Index on Hugging Face, measured on the same sealed v1.6 pool, scorer and cost basis as every other row.",
   "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank.",
   "scorer_source_sha256": "9d512dc13d7d876b233560e8710ed5f64d83f555be90141fa62644f22d5c9421"
  },
  {
   "baseline_categories_sha256": "a2fe49209522d498b3be81e884f9751392c9c135669f11fbb5b8bdda0b2765ce",
   "baseline_systems_sha256": "2ff7130b5dc1946f005044b773c703835d902b808dec9a7f1556f2b9452b2657",
   "date": "2026-10-07",
   "keys": [
    "surogate-rune-26b-a4b-v3-v16",
    "xor-26b-a4b"
   ],
   "label": "a7",
   "note": "Surogate Rune 26B-A4B v3 and Xor 26B-A4B from the top 20 of the community Jev Decision Index on Hugging Face, measured on the same sealed v1.6 pool, scorer and cost basis as every other row.",
   "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank.",
   "scorer_source_sha256": "9d512dc13d7d876b233560e8710ed5f64d83f555be90141fa62644f22d5c9421"
  },
  {
   "baseline_categories_sha256": "1922c9d5f5391b2e280288f4b5414edf24ac82e06b06cb6fb8bda3d34bf6342e",
   "baseline_systems_sha256": "67d9aca6921a4f148b1137b2fcd438e582463b6e09a1e9904bf00a60c0b82812",
   "date": "2026-10-07",
   "keys": [
    "h2o-lightning-4b",
    "kahn1"
   ],
   "label": "a8",
   "note": "H2O-Lightning-4B v1.1 (H2O.ai) and Kahn1 4B (Okura66), two Qwen3.5-4B fine-tunes submitted as JevBench add-requests, measured on the same sealed v1.6 pool, scorer and cost basis as every other row; both are priced with labelled cost estimates (Kahn1's server reports no token usage).",
   "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank.",
   "scorer_source_sha256": "7a5b4d4c7b7ce966c1d7048d5106d50a6a700c8d89372e223ac276f41540be9d"
  },
  {
   "baseline_categories_sha256": "07d69bc9f0638886a19c5d63f2473f9b2677b66950b7d2ea79f0d7c7902f4d69",
   "baseline_systems_sha256": "20b282fdf8597453965eb1bb6fe2d2fd47fbd07963adb5c9da97df862c309318",
   "date": "2026-10-08",
   "keys": [
    "aplomb-1",
    "apus-openjev-v1-35b-a3b",
    "apus-openjev-v1-9b",
    "bb-qwen3.5-4b-lora",
    "classone-gemma4-e2b",
    "classone-qwen3.5-9b",
    "coco-decision-4b",
    "decisio-gemma-4-12b-v080",
    "decisio-gemma-4-12b-v090",
    "diffusion-jev",
    "evalengine-decision-4b",
    "exaone-4.0-1.2b-jev-v0.3",
    "hopper-12b-trained",
    "janus-4b",
    "jul-fast",
    "messier-one-v0.2",
    "seb-9b",
    "spx-cd-omni",
    "tacet-sonata",
    "typecastlm-1.4.0",
    "watersheep",
    "watt-flash-0.1"
   ],
   "label": "a11",
   "note": "Completed current-pool submissions added; historical versions and self-hosted/API conditions are disclosed.",
   "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank.",
   "scorer_source_sha256": "4d2833926c2daf284ce98e1fea6d9223dfc08341905778eb77391734638c27d2"
  },
  {
   "baseline_categories_sha256": "25dddf8fab1b6e240bbe68429f5986f74ef6407ee935adc99972390458c05263",
   "baseline_systems_sha256": "e3b67483db81438b8ec1c8adfab8dc8b12d9ef0ddec32617b0dec1d59766fbb2",
   "date": "2026-10-08",
   "keys": [
    "bobcat-flash-1.2",
    "decider-12b",
    "decider-12b-v1",
    "decisio-gemma-4-31b-v080",
    "mercury-decide",
    "rene-1-31b-fp8",
    "spx-cd-flash",
    "spx-cd-pro"
   ],
   "label": "a12",
   "note": "Approval-required completed candidates; current open/API top-five impact reviewed separately.",
   "rule": "Same frozen v1.6 pool, scorer, G_med and cost basis as the published rows; every published row is byte-identical apart from its rank.",
   "scorer_source_sha256": "4d2833926c2daf284ce98e1fea6d9223dfc08341905778eb77391734638c27d2"
  }
 ],
 "amendments": [
  {
   "date": "2026-10-05",
   "revision": "v1.6.1",
   "text": "5 Oct 2026, v1.6.1 (published rule): hosted/API systems now answer the same full item set as self-hosted systems (1,200 sealed + 300 public). Their earlier answers on the API subsets were reused and only the missing sealed items were sent; every item was logged in the exposure ledger before it was sent. API rows are no longer equated. Consequence: every v1.6.0 sealed item has now been seen by at least two providers, so the whole v1.6.0 sealed draw is retired for future releases; v1.6.2 onward draws fresh items from the reserve."
  },
  {
   "date": "2026-10-05",
   "revision": "v1.6.1",
   "text": "5 Oct 2026, v1.6.1 cost rule (published rule): cost per 1,000 decisions is now measured on one common item set for every system: all items except the 23 very long items that lie outside the Jev reference's accepted input range. Before, a system that processed those items paid for their tokens while a system that refused them did not. Refusals still count against Intelligence exactly as before. Hosted APIs with a per-token tariff are priced from their measured tokens on this common set; carried per-decision prices of self-hosted systems are unchanged. Effect: Sage 1.3.0 USD 0.0247 per 1,000 decisions on the common set (0.0650 on all 1,500 items); Jev 1.13.0 0.0323 (unchanged)."
  }
 ],
 "benchmark": "JevBench",
 "board": {
  "A": {
   "leader_wording": null,
   "markers": [],
   "order": [
    "torchcast-decision-4b-v1.3",
    "singularity-super-decider",
    "launchables-myles-g-12b",
    "h2o-lightning-4b",
    "kettei-fat",
    "wald-26b-a4b",
    "torchcast-decision-4b-v1.2",
    "fastino-glide-api",
    "liquid-d1-ultra",
    "fastino-glide-1.2-open",
    "bobcat-flash-1.2",
    "subq-snap-1.1",
    "h2o-lightning-31b",
    "startlux-decision-v1.1-27b-instant",
    "drex-max",
    "mammoth-1",
    "decisio-gemma-4-12b-tuned-commit",
    "jeff_1_1_medium",
    "phillylabs_philly_decider_v2",
    "pplx-decider-v1-1-27b",
    "nimble-v4-max",
    "chon-1.0-e4b",
    "vllm-sr-d3-turbo",
    "vllm-sr-d3",
    "singularity-super-decider-thinking",
    "knowline-26b-a4b-gen2.5",
    "torchcast-decision-4b-v1.1",
    "subq-snap-preview",
    "knowline-35b-a3b-gen1.5",
    "basal-omni-1.0-ultra",
    "xor-26b-a4b",
    "torchcast-decision-27b",
    "arcbright-elf-1-31b",
    "pplx-decider-v1.2",
    "quyet-1-0-large",
    "nimble-v4-max-open",
    "sage-1.3.0",
    "decider-12b",
    "xevn-4b",
    "kettei-chub",
    "surogate-rune-26b-a4b-v3-v16",
    "jeff_1_0_large",
    "lychee-27b",
    "decisio-gemma-4-31b-v080",
    "decider-12b-v1",
    "vllm-sr-d3-mini",
    "sieve-31b-v2",
    "liquid-d1",
    "vera-large",
    "iota-decide-4b-v0.3",
    "swanone",
    "mercury-decide",
    "winnow-12b",
    "bobcat-1.3",
    "jpt-35b-a3b",
    "chon-1.0-12b-fast",
    "elephant-1",
    "metask_jev_rain_31b_trained",
    "deck31b",
    "cygnet",
    "metask_jev_rain_12b_trained",
    "dopest-1.0-31b",
    "eikos-27b",
    "jev-1.13.0",
    "jeffan-31b-1710",
    "kev-27b",
    "jev-omni",
    "celeris-1-decision",
    "gev-26b-decide",
    "decisio-gemma-4-12b-v080",
    "launchables-myles-g-31b",
    "rsi-jev-v6.1-vl-27b",
    "nagi-enormous",
    "startlux-decision-v1.1-27b-autothink",
    "jev-1.14.0",
    "metask_jev_rain_31b_trained_v3",
    "decisio-gemma-4-12b-v090",
    "bespoke-nimble-9b-v3",
    "jev-27b-autotrust",
    "torchcast-decision-12b",
    "messier-one-v0.4",
    "shisa-de-1",
    "microsoft-decision-1",
    "clef",
    "wity-1",
    "wald-4b-v2-1",
    "hopper-12b-trained",
    "blink-v0-3-26b-a4b-nvfp4",
    "decision-2-0-vega-27b",
    "rene-1-31b-fp8",
    "autojev-27b",
    "openjev-thinking",
    "jebadiah-27b",
    "decider-chat-gemma4-31b",
    "attest-31b-v1.0",
    "ninfer-qwen3.8-flash-next",
    "metask-jev-rain-4b",
    "sage-1.4.0",
    "lumma-fev-27b",
    "respan-span-02",
    "ninfer-qwen3.8-27b",
    "spx-cd-flash",
    "decider-35b-a3b",
    "diffusion-jev",
    "spx-cd-omni",
    "openai-decisions",
    "instinct",
    "hyland-hs1",
    "knowline-31b-gen1",
    "amazon-strands-decider-26b-a4b",
    "turion-jev-31b",
    "knowline-4b-gen5.5",
    "jevone",
    "seb-9b-v2",
    "pplx-decider-flash-v1.2",
    "apus-openjev-v1-9b",
    "aplomb-1",
    "jade",
    "tern-35b-a3b",
    "ajev-gemma4-31b-lm6q",
    "arcbright-elf-1-12b",
    "plumb-4b",
    "spx-cd-pro",
    "messier-one-v0.2",
    "spark-s1-4b-v6",
    "bev-bonsai-27b",
    "clef-flash",
    "quyet-1-0-medium",
    "open-jev-zefan-9b",
    "litjev",
    "nimble-9b",
    "simplejev-qwen3.8-27b-selfhost",
    "qwen35-9b-jev-data-mix-v2",
    "lev",
    "jpt-4b",
    "jev-local",
    "aleria-judge-1.1.1",
    "nimble-v4",
    "decider-4b-v2",
    "jeffan-4b-2010",
    "classone-qwen3.5-9b",
    "vansa-3.4",
    "wald-4b-v2",
    "open-jev-zefan-27b-v1.1",
    "gpt-6-luna-low",
    "knowledgator_gliclass_v4_9b",
    "kushal-gemma4-31b-it-autoloops",
    "metask-jev-4b",
    "kahn1",
    "jevk5-v0.3-4b",
    "gpt-6-luna",
    "deck-4b-v1-0",
    "seb-9b",
    "evalengine-decision-4b",
    "decisor_4b",
    "janus-4b",
    "jebadiah-9b-v2-1",
    "jevk5-v02",
    "amos-1-medium",
    "decision2-lux-9b",
    "verdin-4b",
    "typecastlm-1.4.0",
    "blink-4b",
    "fastino-glide",
    "imajev-9b",
    "jiwo-4b",
    "imajev-4b-rtx5090-a2",
    "uniphore-4b-decider",
    "malkuth-4b",
    "hopper",
    "simplejev-qwen3.6-35b-a3b",
    "decisio-qwen3.6-35b-a3b-v090",
    "clef-omni",
    "jeb-35b-a3b",
    "jqv",
    "coco-decision-4b",
    "decision-4b-v12",
    "manchego-v21-v15",
    "standard_one_3b_sh_v4",
    "raw-qwen3-4b-instruct-2507",
    "standard_one_3b_sh",
    "gpt-5.6-luna",
    "jebadiah-9b-v2",
    "reflex-4b",
    "typecastlm",
    "amos-1-large",
    "iota-decide-4b",
    "decision-4b-v11",
    "standard_one_8b_sh",
    "kev-8b",
    "compass-0.2.0",
    "pjev-2b",
    "instinct-dual-4b",
    "arsenelupin-v1.1",
    "gemini-3.1-flash-lite",
    "standardone-8b",
    "goddragon-arbiter-4b",
    "system-one-open",
    "roudone-decider-4b",
    "semif-qwen3.5-4b",
    "ryotide_qwen9",
    "classone-gemma4-e4b",
    "jev-qwen3.5-9b-base-nvfp4",
    "decisionnode-edge-0.9.2",
    "openjev-sglang",
    "classone-qwen3.5-4b",
    "system-one-sg",
    "jez-large-1.0.0",
    "chon-1.0-12b",
    "ryotide-qwen",
    "noma",
    "raw-qwen3-8b",
    "localjev-qwen3.5-4b",
    "opensourcejev-qwen35-4b-q4km",
    "decision-2b",
    "open-alternative-jev",
    "krino-2b",
    "liquid-d1-3b",
    "open-jev-zefan-2b",
    "scaledown-classify-1",
    "imajev-2b",
    "decider-2b",
    "d1a-gemma4-e4b",
    "nemotron-diffusion-8b",
    "deepseek-flash",
    "kev-4b",
    "decisionnode-edge-mini-0.9.2",
    "malkuth-2b",
    "qwen3-reranker-4b",
    "jiwo-0.8b",
    "zerank-2",
    "vegaml-4b",
    "exaone-4.0-1.2b-jev-v0.3",
    "simplejev-qwen3.8-27b",
    "raw-phi-4-mini",
    "decision-machine-1",
    "simplejev-qwen3.5-0.8b",
    "jeetu-decider-2b",
    "jevact",
    "d1a-gemma4-e2b",
    "smalljev",
    "gutsy-0.8b-v0.3",
    "vega-u-qwen3.5-4b",
    "purplemist-mini-1.0",
    "kev-0.6b",
    "jul-fast",
    "opendecision",
    "classone-gemma4-e2b",
    "fastino-gliner-2-5-decide",
    "deem-0.8-v1",
    "decision-fast",
    "quyet-1-0-small-en",
    "classone-qwen3.5-2b",
    "laya-typed-decisions",
    "vega-u-qwen3.5-0.8b",
    "watersheep",
    "openjev-verdict-1.4",
    "raw-qwen3-0.6b",
    "lev-350m",
    "raw-qwen3-1.7b",
    "kev-0.5b",
    "jez-small-1.0.0",
    "openjev-verdict",
    "quyet-1-0-small",
    "watt-flash-0.1",
    "quyet-1-0-tiny",
    "phocinae-largha-150m-v1",
    "liquid-d1-omni-600m",
    "vegaml-0.8b",
    "open-jev-deberta-v3-large",
    "julia-1",
    "verdict-small",
    "tacet-sonata",
    "laya",
    "mxbai-rerank-base-v2",
    "laya-multilingual",
    "certo",
    "clm-8b",
    "bge-reranker-v2-m3",
    "gte-reranker-modernbert-base",
    "mirror",
    "open-jev-json-canvas-joshuasp",
    "qwen3.8-27b"
   ]
  },
  "B": {
   "leader_wording": null,
   "markers": [],
   "order": [
    "singularity-super-decider",
    "kettei-fat",
    "torchcast-decision-4b-v1.3",
    "h2o-lightning-31b",
    "fastino-glide-api",
    "drex-max",
    "fastino-glide-1.2-open",
    "subq-snap-1.1",
    "singularity-super-decider-thinking",
    "mammoth-1",
    "startlux-decision-v1.1-27b-instant",
    "pplx-decider-v1-1-27b",
    "jeff_1_1_medium",
    "wald-26b-a4b",
    "vllm-sr-d3-turbo",
    "liquid-d1-ultra",
    "vllm-sr-d3",
    "basal-omni-1.0-ultra",
    "subq-snap-preview",
    "quyet-1-0-large",
    "h2o-lightning-4b",
    "arcbright-elf-1-31b",
    "lychee-27b",
    "bobcat-flash-1.2",
    "pplx-decider-v1.2",
    "nimble-v4-max",
    "torchcast-decision-27b",
    "nimble-v4-max-open",
    "jeff_1_0_large",
    "torchcast-decision-4b-v1.2",
    "launchables-myles-g-12b",
    "decisio-gemma-4-31b-v080",
    "knowline-26b-a4b-gen2.5",
    "decisio-gemma-4-12b-tuned-commit",
    "sieve-31b-v2",
    "deck31b",
    "dopest-1.0-31b",
    "sage-1.3.0",
    "vera-large",
    "metask_jev_rain_31b_trained",
    "jeffan-31b-1710",
    "bobcat-1.3",
    "launchables-myles-g-31b",
    "decider-12b",
    "mercury-decide",
    "kettei-chub",
    "knowline-35b-a3b-gen1.5",
    "metask_jev_rain_31b_trained_v3",
    "metask_jev_rain_12b_trained",
    "liquid-d1",
    "phillylabs_philly_decider_v2",
    "eikos-27b",
    "decider-12b-v1",
    "xor-26b-a4b",
    "chon-1.0-e4b",
    "jev-1.13.0",
    "kev-27b",
    "startlux-decision-v1.1-27b-autothink",
    "chon-1.0-12b-fast",
    "winnow-12b",
    "torchcast-decision-4b-v1.1",
    "surogate-rune-26b-a4b-v3-v16",
    "openjev-thinking",
    "wity-1",
    "nagi-enormous",
    "swanone",
    "rsi-jev-v6.1-vl-27b",
    "celeris-1-decision",
    "xevn-4b",
    "elephant-1",
    "jev-1.14.0",
    "torchcast-decision-12b",
    "cygnet",
    "iota-decide-4b-v0.3",
    "attest-31b-v1.0",
    "vllm-sr-d3-mini",
    "jev-omni",
    "rene-1-31b-fp8",
    "sage-1.4.0",
    "bespoke-nimble-9b-v3",
    "decision-2-0-vega-27b",
    "jev-27b-autotrust",
    "jpt-35b-a3b",
    "clef",
    "microsoft-decision-1",
    "decisio-gemma-4-12b-v080",
    "decider-chat-gemma4-31b",
    "jebadiah-27b",
    "respan-span-02",
    "decisio-gemma-4-12b-v090",
    "autojev-27b",
    "messier-one-v0.4",
    "gev-26b-decide",
    "spx-cd-flash",
    "lumma-fev-27b",
    "wald-4b-v2-1",
    "hopper-12b-trained",
    "shisa-de-1",
    "diffusion-jev",
    "metask-jev-rain-4b",
    "blink-v0-3-26b-a4b-nvfp4",
    "ninfer-qwen3.8-27b",
    "ninfer-qwen3.8-flash-next",
    "knowline-31b-gen1",
    "decider-35b-a3b",
    "spx-cd-omni",
    "openai-decisions",
    "turion-jev-31b",
    "hyland-hs1",
    "instinct",
    "amazon-strands-decider-26b-a4b",
    "knowline-4b-gen5.5",
    "jevone",
    "ajev-gemma4-31b-lm6q",
    "apus-openjev-v1-9b",
    "seb-9b-v2",
    "pplx-decider-flash-v1.2",
    "jade",
    "aplomb-1",
    "spx-cd-pro",
    "tern-35b-a3b",
    "arcbright-elf-1-12b",
    "plumb-4b",
    "aleria-judge-1.1.1",
    "simplejev-qwen3.8-27b-selfhost",
    "bev-bonsai-27b",
    "spark-s1-4b-v6",
    "gpt-6-luna-low",
    "messier-one-v0.2",
    "open-jev-zefan-9b",
    "gpt-6-luna",
    "litjev",
    "clef-flash",
    "nimble-9b",
    "quyet-1-0-medium",
    "qwen35-9b-jev-data-mix-v2",
    "jev-local",
    "kushal-gemma4-31b-it-autoloops",
    "classone-qwen3.5-9b",
    "open-jev-zefan-27b-v1.1",
    "lev",
    "jpt-4b",
    "nimble-v4",
    "vansa-3.4",
    "wald-4b-v2",
    "decider-4b-v2",
    "jeffan-4b-2010",
    "knowledgator_gliclass_v4_9b",
    "kahn1",
    "metask-jev-4b",
    "jevk5-v0.3-4b",
    "deck-4b-v1-0",
    "decisor_4b",
    "seb-9b",
    "evalengine-decision-4b",
    "janus-4b",
    "fastino-glide",
    "jebadiah-9b-v2-1",
    "jevk5-v02",
    "amos-1-medium",
    "simplejev-qwen3.6-35b-a3b",
    "decision2-lux-9b",
    "imajev-9b",
    "verdin-4b",
    "typecastlm-1.4.0",
    "blink-4b",
    "jiwo-4b",
    "imajev-4b-rtx5090-a2",
    "uniphore-4b-decider",
    "malkuth-4b",
    "clef-omni",
    "hopper",
    "decisio-qwen3.6-35b-a3b-v090",
    "gpt-5.6-luna",
    "jeb-35b-a3b",
    "jqv",
    "coco-decision-4b",
    "standard_one_3b_sh_v4",
    "manchego-v21-v15",
    "decision-4b-v12",
    "amos-1-large",
    "raw-qwen3-4b-instruct-2507",
    "standard_one_3b_sh",
    "reflex-4b",
    "jebadiah-9b-v2",
    "gemini-3.1-flash-lite",
    "typecastlm",
    "standard_one_8b_sh",
    "iota-decide-4b",
    "decision-4b-v11",
    "kev-8b",
    "compass-0.2.0",
    "pjev-2b",
    "standardone-8b",
    "instinct-dual-4b",
    "openjev-sglang",
    "arsenelupin-v1.1",
    "system-one-open",
    "jev-qwen3.5-9b-base-nvfp4",
    "classone-gemma4-e4b",
    "goddragon-arbiter-4b",
    "ryotide_qwen9",
    "roudone-decider-4b",
    "system-one-sg",
    "semif-qwen3.5-4b",
    "decisionnode-edge-0.9.2",
    "classone-qwen3.5-4b",
    "chon-1.0-12b",
    "jez-large-1.0.0",
    "ryotide-qwen",
    "noma",
    "raw-qwen3-8b",
    "localjev-qwen3.5-4b",
    "opensourcejev-qwen35-4b-q4km",
    "decision-2b",
    "open-alternative-jev",
    "deepseek-flash",
    "krino-2b",
    "liquid-d1-3b",
    "open-jev-zefan-2b",
    "scaledown-classify-1",
    "imajev-2b",
    "decider-2b",
    "nemotron-diffusion-8b",
    "d1a-gemma4-e4b",
    "kev-4b",
    "decisionnode-edge-mini-0.9.2",
    "simplejev-qwen3.8-27b",
    "qwen3-reranker-4b",
    "malkuth-2b",
    "jiwo-0.8b",
    "zerank-2",
    "vegaml-4b",
    "exaone-4.0-1.2b-jev-v0.3",
    "raw-phi-4-mini",
    "decision-machine-1",
    "simplejev-qwen3.5-0.8b",
    "jeetu-decider-2b",
    "jevact",
    "d1a-gemma4-e2b",
    "smalljev",
    "gutsy-0.8b-v0.3",
    "vega-u-qwen3.5-4b",
    "purplemist-mini-1.0",
    "kev-0.6b",
    "classone-gemma4-e2b",
    "jul-fast",
    "opendecision",
    "fastino-gliner-2-5-decide",
    "deem-0.8-v1",
    "decision-fast",
    "quyet-1-0-small-en",
    "classone-qwen3.5-2b",
    "laya-typed-decisions",
    "vega-u-qwen3.5-0.8b",
    "watersheep",
    "raw-qwen3-0.6b",
    "openjev-verdict-1.4",
    "raw-qwen3-1.7b",
    "lev-350m",
    "kev-0.5b",
    "jez-small-1.0.0",
    "openjev-verdict",
    "quyet-1-0-small",
    "watt-flash-0.1",
    "quyet-1-0-tiny",
    "phocinae-largha-150m-v1",
    "liquid-d1-omni-600m",
    "vegaml-0.8b",
    "open-jev-deberta-v3-large",
    "julia-1",
    "verdict-small",
    "tacet-sonata",
    "laya",
    "mxbai-rerank-base-v2",
    "laya-multilingual",
    "certo",
    "clm-8b",
    "bge-reranker-v2-m3",
    "gte-reranker-modernbert-base",
    "mirror",
    "open-jev-json-canvas-joshuasp",
    "qwen3.8-27b"
   ]
  },
  "C": {
   "leader_wording": null,
   "markers": [],
   "order": [
    "torchcast-decision-4b-v1.3",
    "singularity-super-decider",
    "h2o-lightning-4b",
    "kettei-fat",
    "wald-26b-a4b",
    "torchcast-decision-4b-v1.2",
    "fastino-glide-api",
    "liquid-d1-ultra",
    "fastino-glide-1.2-open",
    "bobcat-flash-1.2",
    "subq-snap-1.1",
    "h2o-lightning-31b",
    "startlux-decision-v1.1-27b-instant",
    "drex-max",
    "mammoth-1",
    "decisio-gemma-4-12b-tuned-commit",
    "jeff_1_1_medium",
    "pplx-decider-v1-1-27b",
    "nimble-v4-max",
    "vllm-sr-d3-turbo",
    "vllm-sr-d3",
    "singularity-super-decider-thinking",
    "knowline-26b-a4b-gen2.5",
    "subq-snap-preview",
    "knowline-35b-a3b-gen1.5",
    "basal-omni-1.0-ultra",
    "torchcast-decision-27b",
    "arcbright-elf-1-31b",
    "pplx-decider-v1.2",
    "quyet-1-0-large",
    "nimble-v4-max-open",
    "sage-1.3.0",
    "decider-12b",
    "kettei-chub",
    "jeff_1_0_large",
    "lychee-27b",
    "decisio-gemma-4-31b-v080",
    "sieve-31b-v2",
    "liquid-d1",
    "vera-large",
    "mercury-decide",
    "bobcat-1.3",
    "decider-12b-v1",
    "chon-1.0-12b-fast",
    "metask_jev_rain_31b_trained",
    "deck31b",
    "metask_jev_rain_12b_trained",
    "dopest-1.0-31b",
    "eikos-27b",
    "jev-1.13.0",
    "jeffan-31b-1710",
    "kev-27b",
    "launchables-myles-g-12b",
    "winnow-12b",
    "launchables-myles-g-31b",
    "rsi-jev-v6.1-vl-27b",
    "nagi-enormous",
    "startlux-decision-v1.1-27b-autothink",
    "metask_jev_rain_31b_trained_v3",
    "jev-1.14.0",
    "torchcast-decision-12b",
    "wity-1",
    "decision-2-0-vega-27b",
    "rene-1-31b-fp8",
    "openjev-thinking",
    "celeris-1-decision",
    "xor-26b-a4b",
    "attest-31b-v1.0",
    "sage-1.4.0",
    "respan-span-02",
    "clef",
    "phillylabs_philly_decider_v2",
    "decider-chat-gemma4-31b",
    "surogate-rune-26b-a4b-v3-v16",
    "chon-1.0-e4b",
    "swanone",
    "spx-cd-flash",
    "microsoft-decision-1",
    "jev-27b-autotrust",
    "jebadiah-27b",
    "elephant-1",
    "bespoke-nimble-9b-v3",
    "jev-omni",
    "torchcast-decision-4b-v1.1",
    "knowline-31b-gen1",
    "cygnet",
    "turion-jev-31b",
    "autojev-27b",
    "lumma-fev-27b",
    "openai-decisions",
    "iota-decide-4b-v0.3",
    "xevn-4b",
    "diffusion-jev",
    "decisio-gemma-4-12b-v080",
    "decisio-gemma-4-12b-v090",
    "jpt-35b-a3b",
    "messier-one-v0.4",
    "vllm-sr-d3-mini",
    "ajev-gemma4-31b-lm6q",
    "wald-4b-v2-1",
    "hopper-12b-trained",
    "ninfer-qwen3.8-27b",
    "gev-26b-decide",
    "metask-jev-rain-4b",
    "shisa-de-1",
    "blink-v0-3-26b-a4b-nvfp4",
    "spx-cd-pro",
    "hyland-hs1",
    "ninfer-qwen3.8-flash-next",
    "decider-35b-a3b",
    "spx-cd-omni",
    "aleria-judge-1.1.1",
    "instinct",
    "amazon-strands-decider-26b-a4b",
    "simplejev-qwen3.8-27b-selfhost",
    "knowline-4b-gen5.5",
    "jevone",
    "gpt-6-luna-low",
    "seb-9b-v2",
    "pplx-decider-flash-v1.2",
    "apus-openjev-v1-9b",
    "gpt-6-luna",
    "aplomb-1",
    "kushal-gemma4-31b-it-autoloops",
    "jade",
    "tern-35b-a3b",
    "arcbright-elf-1-12b",
    "plumb-4b",
    "messier-one-v0.2",
    "spark-s1-4b-v6",
    "bev-bonsai-27b",
    "clef-flash",
    "quyet-1-0-medium",
    "open-jev-zefan-9b",
    "litjev",
    "open-jev-zefan-27b-v1.1",
    "nimble-9b",
    "qwen35-9b-jev-data-mix-v2",
    "lev",
    "jpt-4b",
    "fastino-glide",
    "jev-local",
    "nimble-v4",
    "decider-4b-v2",
    "jeffan-4b-2010",
    "classone-qwen3.5-9b",
    "vansa-3.4",
    "wald-4b-v2",
    "knowledgator_gliclass_v4_9b",
    "metask-jev-4b",
    "kahn1",
    "jevk5-v0.3-4b",
    "deck-4b-v1-0",
    "seb-9b",
    "evalengine-decision-4b",
    "decisor_4b",
    "janus-4b",
    "simplejev-qwen3.6-35b-a3b",
    "jebadiah-9b-v2-1",
    "jevk5-v02",
    "amos-1-medium",
    "decision2-lux-9b",
    "verdin-4b",
    "typecastlm-1.4.0",
    "blink-4b",
    "gpt-5.6-luna",
    "imajev-9b",
    "jiwo-4b",
    "imajev-4b-rtx5090-a2",
    "amos-1-large",
    "uniphore-4b-decider",
    "malkuth-4b",
    "hopper",
    "decisio-qwen3.6-35b-a3b-v090",
    "clef-omni",
    "jeb-35b-a3b",
    "jqv",
    "coco-decision-4b",
    "decision-4b-v12",
    "manchego-v21-v15",
    "standard_one_3b_sh_v4",
    "gemini-3.1-flash-lite",
    "raw-qwen3-4b-instruct-2507",
    "standard_one_3b_sh",
    "jebadiah-9b-v2",
    "reflex-4b",
    "typecastlm",
    "iota-decide-4b",
    "decision-4b-v11",
    "standard_one_8b_sh",
    "kev-8b",
    "compass-0.2.0",
    "pjev-2b",
    "instinct-dual-4b",
    "arsenelupin-v1.1",
    "standardone-8b",
    "goddragon-arbiter-4b",
    "system-one-open",
    "roudone-decider-4b",
    "semif-qwen3.5-4b",
    "ryotide_qwen9",
    "classone-gemma4-e4b",
    "jev-qwen3.5-9b-base-nvfp4",
    "decisionnode-edge-0.9.2",
    "openjev-sglang",
    "classone-qwen3.5-4b",
    "system-one-sg",
    "jez-large-1.0.0",
    "chon-1.0-12b",
    "ryotide-qwen",
    "noma",
    "raw-qwen3-8b",
    "localjev-qwen3.5-4b",
    "opensourcejev-qwen35-4b-q4km",
    "decision-2b",
    "deepseek-flash",
    "open-alternative-jev",
    "krino-2b",
    "liquid-d1-3b",
    "open-jev-zefan-2b",
    "scaledown-classify-1",
    "imajev-2b",
    "decider-2b",
    "d1a-gemma4-e4b",
    "nemotron-diffusion-8b",
    "kev-4b",
    "decisionnode-edge-mini-0.9.2",
    "malkuth-2b",
    "qwen3-reranker-4b",
    "jiwo-0.8b",
    "zerank-2",
    "vegaml-4b",
    "simplejev-qwen3.8-27b",
    "exaone-4.0-1.2b-jev-v0.3",
    "raw-phi-4-mini",
    "decision-machine-1",
    "simplejev-qwen3.5-0.8b",
    "jeetu-decider-2b",
    "jevact",
    "d1a-gemma4-e2b",
    "smalljev",
    "gutsy-0.8b-v0.3",
    "vega-u-qwen3.5-4b",
    "purplemist-mini-1.0",
    "kev-0.6b",
    "jul-fast",
    "opendecision",
    "classone-gemma4-e2b",
    "fastino-gliner-2-5-decide",
    "deem-0.8-v1",
    "decision-fast",
    "quyet-1-0-small-en",
    "classone-qwen3.5-2b",
    "laya-typed-decisions",
    "vega-u-qwen3.5-0.8b",
    "watersheep",
    "openjev-verdict-1.4",
    "raw-qwen3-0.6b",
    "lev-350m",
    "raw-qwen3-1.7b",
    "kev-0.5b",
    "jez-small-1.0.0",
    "openjev-verdict",
    "quyet-1-0-small",
    "watt-flash-0.1",
    "quyet-1-0-tiny",
    "phocinae-largha-150m-v1",
    "liquid-d1-omni-600m",
    "vegaml-0.8b",
    "open-jev-deberta-v3-large",
    "julia-1",
    "verdict-small",
    "tacet-sonata",
    "laya",
    "mxbai-rerank-base-v2",
    "laya-multilingual",
    "certo",
    "clm-8b",
    "bge-reranker-v2-m3",
    "gte-reranker-modernbert-base",
    "mirror",
    "open-jev-json-canvas-joshuasp",
    "qwen3.8-27b"
   ]
  },
  "capability": {
   "order": [
    "gpt-6-luna",
    "gpt-6-luna-low",
    "gpt-5.6-luna",
    "deepseek-flash",
    "kettei-fat",
    "singularity-super-decider-thinking",
    "mammoth-1",
    "knowline-31b-gen1",
    "fastino-glide-1.2-open",
    "singularity-super-decider",
    "fastino-glide-api",
    "h2o-lightning-31b",
    "drex-max",
    "pplx-decider-v1-1-27b",
    "vllm-sr-d3-turbo",
    "startlux-decision-v1.1-27b-instant",
    "vllm-sr-d3",
    "basal-omni-1.0-ultra",
    "dopest-1.0-31b",
    "subq-snap-1.1",
    "jeff_1_1_medium",
    "arcbright-elf-1-31b",
    "quyet-1-0-large",
    "jeffan-31b-1710",
    "jeff_1_0_large",
    "subq-snap-preview",
    "ajev-gemma4-31b-lm6q",
    "nimble-v4-max-open",
    "sieve-31b-v2",
    "turion-jev-31b",
    "pplx-decider-v1.2",
    "liquid-d1-ultra",
    "torchcast-decision-27b",
    "lychee-27b",
    "nimble-v4-max",
    "metask_jev_rain_31b_trained_v3",
    "decisio-gemma-4-31b-v080",
    "vera-large",
    "kev-27b",
    "metask_jev_rain_31b_trained",
    "launchables-myles-g-31b",
    "startlux-decision-v1.1-27b-autothink",
    "sage-1.3.0",
    "spx-cd-pro",
    "amos-1-large",
    "bobcat-1.3",
    "deck31b",
    "torchcast-decision-4b-v1.3",
    "knowline-26b-a4b-gen2.5",
    "eikos-27b",
    "jev-1.13.0",
    "fastino-glide",
    "wity-1",
    "metask_jev_rain_12b_trained",
    "aleria-judge-1.1.1",
    "wald-26b-a4b",
    "knowline-35b-a3b-gen1.5",
    "rene-1-31b-fp8",
    "openjev-thinking",
    "decisio-gemma-4-12b-tuned-commit",
    "clef",
    "jev-1.14.0",
    "h2o-lightning-4b",
    "simplejev-qwen3.8-27b-selfhost",
    "liquid-d1",
    "spx-cd-flash",
    "jev-27b-autotrust",
    "nagi-enormous",
    "mercury-decide",
    "celeris-1-decision",
    "openai-decisions",
    "surogate-rune-26b-a4b-v3-v16",
    "rsi-jev-v6.1-vl-27b",
    "sage-1.4.0",
    "kettei-chub",
    "bobcat-flash-1.2",
    "kushal-gemma4-31b-it-autoloops",
    "torchcast-decision-4b-v1.2",
    "swanone",
    "lumma-fev-27b",
    "torchcast-decision-12b",
    "autojev-27b",
    "simplejev-qwen3.8-27b",
    "decider-12b",
    "decision-2-0-vega-27b",
    "jev-omni",
    "winnow-12b",
    "xor-26b-a4b",
    "decider-12b-v1",
    "cygnet",
    "microsoft-decision-1",
    "decisio-gemma-4-12b-v080",
    "jpt-35b-a3b",
    "launchables-myles-g-12b",
    "instinct",
    "decider-chat-gemma4-31b",
    "elephant-1",
    "jebadiah-27b",
    "gev-26b-decide",
    "chon-1.0-12b-fast",
    "decisio-gemma-4-12b-v090",
    "blink-v0-3-26b-a4b-nvfp4",
    "ninfer-qwen3.8-flash-next",
    "attest-31b-v1.0",
    "respan-span-02",
    "phillylabs_philly_decider_v2",
    "wald-4b-v2-1",
    "jevone",
    "vllm-sr-d3-mini",
    "torchcast-decision-4b-v1.1",
    "iota-decide-4b-v0.3",
    "xevn-4b",
    "chon-1.0-e4b",
    "simplejev-qwen3.6-35b-a3b",
    "hopper-12b-trained",
    "spx-cd-omni",
    "ninfer-qwen3.8-27b",
    "open-jev-zefan-27b-v1.1",
    "aplomb-1",
    "bespoke-nimble-9b-v3",
    "amazon-strands-decider-26b-a4b",
    "litjev",
    "tern-35b-a3b",
    "knowline-4b-gen5.5",
    "decider-35b-a3b",
    "bev-bonsai-27b",
    "metask-jev-rain-4b",
    "shisa-de-1",
    "plumb-4b",
    "arcbright-elf-1-12b",
    "decider-4b-v2",
    "seb-9b-v2",
    "hyland-hs1",
    "jevk5-v0.3-4b",
    "nimble-v4",
    "jeffan-4b-2010",
    "clef-flash",
    "jpt-4b",
    "vansa-3.4",
    "deck-4b-v1-0",
    "jebadiah-9b-v2-1",
    "knowledgator_gliclass_v4_9b",
    "seb-9b",
    "janus-4b",
    "amos-1-medium",
    "jevk5-v02",
    "messier-one-v0.4",
    "imajev-9b",
    "imajev-4b-rtx5090-a2",
    "jqv",
    "open-jev-zefan-9b",
    "metask-jev-4b",
    "kahn1",
    "pplx-decider-flash-v1.2",
    "lev",
    "decision2-lux-9b",
    "jeb-35b-a3b",
    "decisio-qwen3.6-35b-a3b-v090",
    "blink-4b",
    "hopper",
    "malkuth-4b",
    "qwen35-9b-jev-data-mix-v2",
    "decision-4b-v12",
    "jiwo-4b",
    "standard_one_8b_sh",
    "gemini-3.1-flash-lite",
    "verdin-4b",
    "wald-4b-v2",
    "quyet-1-0-medium",
    "evalengine-decision-4b",
    "reflex-4b",
    "openjev-sglang",
    "standardone-8b",
    "clef-omni",
    "manchego-v21-v15",
    "messier-one-v0.2",
    "diffusion-jev",
    "jev-local",
    "instinct-dual-4b",
    "decision-4b-v11",
    "jev-qwen3.5-9b-base-nvfp4",
    "iota-decide-4b",
    "uniphore-4b-decider",
    "jebadiah-9b-v2",
    "coco-decision-4b",
    "apus-openjev-v1-9b",
    "roudone-decider-4b",
    "typecastlm",
    "nimble-9b",
    "pjev-2b",
    "typecastlm-1.4.0",
    "ryotide-qwen",
    "compass-0.2.0",
    "standard_one_3b_sh_v4",
    "standard_one_3b_sh",
    "semif-qwen3.5-4b",
    "arsenelupin-v1.1",
    "localjev-qwen3.5-4b",
    "decision-2b",
    "jez-large-1.0.0",
    "noma",
    "spark-s1-4b-v6",
    "imajev-2b",
    "ryotide_qwen9",
    "scaledown-classify-1",
    "goddragon-arbiter-4b",
    "decisionnode-edge-0.9.2",
    "system-one-open",
    "d1a-gemma4-e4b",
    "classone-qwen3.5-9b",
    "kev-8b",
    "liquid-d1-3b",
    "zerank-2",
    "jade",
    "open-alternative-jev",
    "qwen3.8-27b",
    "opensourcejev-qwen35-4b-q4km",
    "open-jev-zefan-2b",
    "exaone-4.0-1.2b-jev-v0.3",
    "malkuth-2b",
    "decisionnode-edge-mini-0.9.2",
    "nemotron-diffusion-8b",
    "jiwo-0.8b",
    "decision-machine-1",
    "decisor_4b",
    "d1a-gemma4-e2b",
    "qwen3-reranker-4b",
    "decider-2b",
    "laya-typed-decisions",
    "bge-reranker-v2-m3",
    "kev-4b",
    "mxbai-rerank-base-v2",
    "lev-350m",
    "jez-small-1.0.0",
    "certo",
    "purplemist-mini-1.0",
    "chon-1.0-12b",
    "smalljev",
    "vega-u-qwen3.5-4b",
    "jul-fast",
    "krino-2b",
    "gutsy-0.8b-v0.3",
    "gte-reranker-modernbert-base",
    "jeetu-decider-2b",
    "watt-flash-0.1",
    "decision-fast",
    "openjev-verdict-1.4",
    "quyet-1-0-small-en",
    "classone-gemma4-e4b",
    "classone-qwen3.5-4b",
    "opendecision",
    "verdict-small",
    "kev-0.6b",
    "vegaml-4b",
    "vega-u-qwen3.5-0.8b",
    "watersheep",
    "open-jev-deberta-v3-large",
    "raw-phi-4-mini",
    "fastino-gliner-2-5-decide",
    "raw-qwen3-4b-instruct-2507",
    "vegaml-0.8b",
    "kev-0.5b",
    "liquid-d1-omni-600m",
    "laya",
    "quyet-1-0-small",
    "quyet-1-0-tiny",
    "simplejev-qwen3.5-0.8b",
    "open-jev-json-canvas-joshuasp",
    "phocinae-largha-150m-v1",
    "raw-qwen3-8b",
    "system-one-sg",
    "jevact",
    "openjev-verdict",
    "mirror",
    "tacet-sonata",
    "clm-8b",
    "classone-qwen3.5-2b",
    "deem-0.8-v1",
    "laya-multilingual",
    "classone-gemma4-e2b",
    "julia-1",
    "raw-qwen3-1.7b",
    "raw-qwen3-0.6b"
   ]
  }
 },
 "bootstrap": {
  "B": 1000,
  "bootstrap_seed": 16,
  "g_med_fixed": true
 },
 "headline": "A",
 "label": "JevBench v1.8.0: GPU-time cost basis",
 "measurement_notes": [
  "Method v1.7: from v1.7.0 a decision that takes longer than 10 seconds (the measured time, strictly more than 10.000 s) counts as a wrong answer and as zero calibration for that item (each item type's calibration is multiplied by the share of its decisions answered within 10 s). Every row is re-scored from its recorded per-decision times: Intelligence, Calibration, Capability, the composites and their intervals are recomputed with the same frozen scorer plus the rule; the Speed axis now uses all decisions given (it replaces the 124-item subset of v1.6). Cost, tokens, the draws, the raw outputs and the field median are unchanged; no model was run again for the re-scored rows. Every re-scored row shows its v1.6 values next to the new ones.",
  "Time limit on measured time: the limit uses the measured latency, not the displayed estimate (x2 + 0.15 s for self-hosted rows), so a self-hosted row can show an adjusted p95 above 10 s with no decision over the limit. In total 1,455 decisions across 59 rows are over the limit.",
  "Re-measure pending: 5 rows (chon-1.0-e4b, nev-2b, modernbert-jev-v2, mghafiri-qwen3.5-0.8b-decision-model, reflex-instinct-0.6b) recorded their per-decision times on a loaded CPU host, so they cannot be scored under the time limit and no estimate is published. They stay on the board with their v1.6 values, labelled as v1.6, unranked, until they are measured again.",
  "Re-measured in v1.7.0: 4 rows that were pending (loaded CPU host) were measured again on the same items on a dedicated GPU. Each GPU row replaces the earlier CPU row in full, scores and times; GPU times are never combined with CPU scores, and the CPU values stay visible as the row's v1.6 values. pjev-2b (Kailune AI, frozen MiniCPM5-2B + merged LoRA r64, letter-logit readout, T=1 stub) (pjev-2b): no decision took longer than 10 seconds; re-measured on the same items, second and last use of these weights on the draw. PurpleMIST-Mini-1.0 (Olaverse) (purplemist-mini-1.0): no decision took longer than 10 seconds; re-measured on the same items, second and last use of these weights on the draw. Vega-U (Qwen3.5-4B reader) (vega-u-qwen3.5-4b): 15 of its 1,500 decisions took longer than 10 seconds; re-measured on the same items, second and last use of these weights on the draw. Verdin-4B (Victor Bennett, Qwen3.5-4B + merged LoRA r16 + pointer head) (verdin-4b): no decision took longer than 10 seconds; re-measured on the same items, second and last use of these weights on the draw.",
  "Open/sealed gap penalty: the public-versus-sealed gap penalty is computed from the row's own results, so a decision that now fails can slightly reduce that penalty and raise Intelligence a little. This affects 1 row in this release (shisa-de-1 (shisa-ai, Gemma-4-26B-A4B): +0.006 Intelligence points); it is a side effect of the existing penalty, not a change to it.",
  "Dated carry: 14 rows measured before v1.6 (v1.5 method) are carried as published and not re-scored.",
  "Equated API rows: the pool re-runs behind the A4 and A5 equating offsets are scored under the same rule, so the offsets moved with them. These rows keep the 2-decimal precision of their earlier publication.",
  "New in v1.7.0: 26 rows measured for the first time under the 10 s rule. AJev lm6q (Gemma 4 31B) (ajev-gemma4-31b-lm6q): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Aleria Judge 1.1.1 (ALERIA-ML, inference technique on stock Gemma 4 31B, author server, self-hosted) (aleria-judge-1.1.1): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 15 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. Arcbright Elf 1 31B (arcbright-elf-1-31b): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Bobcat 1.3 (Gemma 4 31B) (bobcat-1.3): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 15 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. DecisionNode Edge 0.9.2 (decisionnode-edge-0.9.2): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. DecisionNode Edge Mini 0.9.2 (decisionnode-edge-mini-0.9.2): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. baselabs elephant-1 (9B) (elephant-1): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Fastino GLiDE 1.2 (open weights, self-hosted) (fastino-glide-1.2-open): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Jeffan-31b-1710 (jeffan-31b-1710): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 15 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. Jeffan-4B-2010 (jeffan-4b-2010): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Jez-large-1.0.0 (pragmaticfrontier, Gemma-4-E4B label-logit decision model, author server, self-hosted) (jez-large-1.0.0): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Jez-small-1.0.0 (pragmaticfrontier, Qwen3.5-0.8B label-logit decision model, author server, self-hosted) (jez-small-1.0.0): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. KnowLine 31B-Gen1 (knowline-31b-gen1): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 15 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. KnowLine 35B-A3B-Gen1.5 (knowline-35b-a3b-gen1.5): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. FrontiersMind Lumma-fev-27B (lumma-fev-27b): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. baselabs mammoth-1 (27B) (mammoth-1): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Nimble-v4-max (open weights, self-hosted) (nimble-v4-max-open): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 1 of its 1,500 decision took longer than 10 seconds and is scored by the rule. Its mandatory disclosure and licence note are shown on the row. Seb-9B-v2 (ironbcc, Qwen3.5-9B fine-tune, one forward pass, stock vLLM, self-hosted) (seb-9b-v2): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Super Decider (singularity-super-decider): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 11 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. Super Decider (thinking) (singularity-super-decider-thinking): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 4 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. Standard One 3B SH v4 (Standard Thinking) (standard_one_3b_sh_v4): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Tern-35B-A3B (matthewagi, Qwen3.6 35B-A3B MoE decision model, author server, self-hosted) (tern-35b-a3b): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Vera Large (vera-large): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. vLLM-SR d3-mini (vllm-sr-d3-mini): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; 15 of its 1,500 decisions took longer than 10 seconds and are scored by the rule. Its mandatory disclosure and licence note are shown on the row. Wald 26B-A4B (Gemma 4 26B-A4B) (wald-26b-a4b): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row. Xevn-4B (xevn-4b): measured for the first time under the 10 s rule, so it has no v1.6 values and no old-versus-new comparison; no decision took longer than 10 seconds. Its mandatory disclosure and licence note are shown on the row.",
  "Board effect: the 10 s rule (method v1.7) re-scored every row; the merged-board top fives before/after v1.7.0 were compared over all 36 mixed, open-weights and API variants (including cost caps of 7x and 10x the reference cost, each also with no latency cap). The reference cost is that of Jev 1.13.0, USD 0.0323 per 1,000 decisions, and the official cap is 2x that. Together with 4 release-board lists, 37 of 40 top-five lists change. Top-five changes caused by: the 10 s rule (method v1.7) and the new rows arcbright-elf-1-31b, fastino-glide-1.2-open, knowline-31b-gen1, mammoth-1, singularity-super-decider, singularity-super-decider-thinking (measured for the first time in v1.7.0); entering a top five: arcbright-elf-1-31b, fastino-glide-1.2-open, kettei-fat, knowline-31b-gen1, mammoth-1, singularity-super-decider, singularity-super-decider-thinking; leaving a top five: chon-1.0-12b, drex-max, fastino-glide-api, h2o-lightning-31b, liquid-d1-ultra, nimble-v4-max, qwen3.8-27b, subq-snap-1.1, subq-snap-preview, vllm-sr-d3, vllm-sr-d3-turbo. ",
  "Publication of v1.7.0 was approved on 24 Oct 2026 after review of the top-five preview.",
  "Three new rows on v1.6-regular-20261012: Amazon Strands Decider 26B-A4B, Turion JEV 31B and Chon-1.0-12B-Fast. No method change. All 285 carried rows retain every score, confidence interval and diagnostic; only placements move. New rows use their already-scored full-field B=1000 intervals and the 10-second share rule.",
  "One new row on v1.6-regular-20261012: Jeetu Decider 2B. No method change. All 288 carried rows retain every score, confidence interval and diagnostic; only placements move. The new row uses its already-scored full-field B=1000 intervals and the 10-second share rule.",
  "Chon-1.0-E4B GPU scores and times replace its CPU measurement in full; classify-1 (ScaleDown) added as one new hosted API row; no method change. All 288 other rows retain every score, confidence interval and diagnostic; only placements move.",
  "One new self-hosted row DOPEST 1.0 (31B); no method change. All 290 carried rows retain every score, confidence interval and diagnostic; only placements move.",
  "One new self-hosted row Sieve-31B-v2; no method change. All 291 carried rows retain every score, confidence interval and diagnostic; only placements move.",
  "One new self-hosted row Decisio 12B tuned commit (variant of the Decisio family); no method change. All 292 carried rows retain every score, confidence interval and diagnostic; only placements move.",
  "Two new self-hosted rows D1A on Gemma 4 E4B (JohnP1 d1a-e4b) and D1A on Gemma 4 E2B (JohnP1 d1a-e2b); no method change. On each D1A row, 15 of 1,500 answers were made on a prompt truncated at the author's 65,536-token limit and took over 10 s (up to 20.6 s on E4B and 16.2 s on E2B); the v1.7 share view counts them as wrong.",
  "One new self-hosted row Jeff-1.1-Medium; no method change. Question-aware retrieval answered 15 long items within 8,192 input tokens; zero of 1,500 answers exceeded 10 s. Cost is an ESTIMATE; an uncounted second-prefill estimate of 427,166 input tokens is not priced. Weights CC BY-NC-4.0 (non-commercial).",
  "One new self-hosted row GLiClass v4 9B; Myles-G 12B now priced at its public list price; Super Decider links point to the vendor page; no method change. GLiClass answered 1,485 of 1,500 requests; 15 HTTP 422 length refusals count as misses. Its cost is an ESTIMATE from Qwen/Qwen3.5-9B with zero generated output.",
  "Three new self-hosted rows Philly Decider v2, iota-decide-4b-v0.3 and Decisio 0.9.0 (Qwen3.6-35B-A3B); no method change. Super Decider rows now link to the vendor page (completes the v1.7.9 change). Philly and iota v0.3 answered all 1,500 requests with disclosed truncation. Decisio answered 1,485; 15 HTTP 500 over-length refusals count as misses. All three costs are ESTIMATE base-model prices; Decisio uses disclosed offline engine-input billing and exceeds the cost cap.",
  "Cost correction for four Decisio Gemma rows (engine-processed input); no new measurement, no method change.",
  "Cost-basis wording clarification for Jez Large, Jez Small and D1A Gemma 4; no measurements, numeric costs, scores, ranks, or method changed.",
  "First release on the 27 Oct regular draw: H2O-Lightning-4B v1.4.1 (paid evaluation), shown unranked until the draw is equated to the reference scale. It answered all 1,500 requests of the new draw (5 long states trimmed by the model at pre-registered positions), none over 10 s. Its draw is not yet equated to the reference scale, so its numbers are shown as published (offset 0) with the reference gap-penalty median. Cost is an ESTIMATE from the base model price on the prompt tokens the model processed. This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own draw. NOT RANKED: Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale. This draw differs strongly from earlier draws (72% multilingual items). On the 300 public items every draw shares, this version scores 57.2 Intelligence vs 61.1 for H2O-Lightning-4B v1.1; its higher overall Intelligence comes from this draw's sealed items. Shown unranked until the draw is equated. No existing row, rank or ranking changed.",
  "Five more rows on the 27 Oct regular draw (Torchcast Decision 31B v1.0, paid evaluation; Arcbright Elf 1 12B (r2); Jeff-1.1-Qwen-4B; Decisio 12B tuned commit-think; Jeff-1.1-Large), shown unranked until the draw is equated to the reference scale. Each answered all 1,500 requests of the draw (the draw's 5 long states shortened by each model's own documented rule and answered), none over 10 s. Their draw is not yet equated to the reference scale, so their numbers are shown as published (offset 0) with the reference gap-penalty median. Cost is an ESTIMATE from the base model price on the prompt tokens each model processed, over all 1,500 items. This draw has no subject-topic labels yet, so the rows have no topic cells; their use-case and language cells are measured on their own draw. NOT RANKED: Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale. This draw differs strongly from earlier draws (72% multilingual items). Shown unranked until the draw is equated. No existing score, rank or ranking changed. Corrected the DOPEST 1.0 31B notes: its serving-code repository has been public throughout.",
  "Added Microsoft-Decision-1 with updated weights (paid evaluation) on the 27 Oct regular draw; that draw's rows stay unranked until stronger anchors are measured. Microsoft-Decision-1 (updated weights) answered 1,489 of 1,493 items of the draw: 7 items its provider had seen in an earlier evaluation are excluded, and 4 requests over its 32,768-token context limit count as wrong; none took longer than 10 s. The served model id is unchanged from the first measurement; the weights update is Microsoft's statement. Its cost uses the advertised Azure Foundry tariff (USD 0.042 per million input tokens, output free), as for the first Microsoft-Decision-1 row, which stays unchanged. The draw is not equated to the reference scale, so its numbers are shown as published (offset 0) with the reference gap-penalty median. This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own 1,493 items. NOT RANKED: every row on that draw now carries the note \"Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale: its first anchor runs used models weaker than this row, and stronger anchors are being measured. Shown unranked until then.\" No existing score, rank or ranking changed.",
  "Added Strike-1 (hosted) on the 27 Oct regular draw, shown unranked; costs on that draw now use the items every row answered. Strike-1 is a hosted decision API (EU, OVH Gravelines); it answered 1,495 of the draw's 1,500 items: 5 items over the provider's input limit were refused and count as wrong; none took longer than 10 s. The vendor reports 0 output tokens (no text is generated) and describes the model as a fine-tune of Qwen/Qwen3.5-4B, whose market reference prices its cost (estimate). SAME-ITEMS COST: Microsoft-Decision-1 (updated weights) ran on 1,493 items and answered 1,489; Strike-1's 5 refused long items are among those it did not answer, so every row on this draw is costed on those 1,489 items, with its own price reference and token basis; its all-items cost is stated next to it. Only cost, the Cost axis and the composites of those rows change; Intelligence, Calibration, Speed and every category cell are unchanged. The draw is not equated to the reference scale, so its numbers are shown as published (offset 0). NOT RANKED: the carried rows on that draw keep their notes, based on \"Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale: its first anchor runs used models weaker than this row, and stronger anchors are being measured. Shown unranked until then.\" Strike-1 carries the note \"Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale; stronger anchor runs for that draw are being measured. Shown unranked until then.\" No existing rank or ranking changed.",
  "Added Jev-Style-2B (self-hosted) on the 27 Oct regular draw, shown unranked. FIRST MEASUREMENT of Jev-Style-2B, a full fine-tune of Qwen/Qwen3.5-2B (Apache-2.0). TRAINING: trained in part on soft labels distilled from TypeSafe Jev 1.14 via its public API (334,739 rows), plus official train splits of several public benchmarks (not zero-shot); the author states that public JevBench items were not used. We did not inspect the training examples or verify overlap. SERVING: measured through the author's own documented HTTP server on CUDA, bf16, CUDA graphs and the FLA kernel; the author's own 74.9% JevBench figure used float32 on Apple MPS and is not an official result. The multimodal tower is present but unused on this text draw. Contributor code ran only on a disposable rented pod. ITEMS: all 1,500 items of the draw were sent; the 5 longest items exceed the model's 32,768-token input limit and were refused (HTTP 400), scored as misses and kept in the denominator. TOKENS: output tokens reported as 0 (native probability distribution readout, no text generation); input is the server's native usage. DRAW: measured on the 27 Oct 2026 regular draw (fresh sealed items; not yet equated to the reference scale, so its numbers are shown as published, offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own draw. LATENCY: p50 0.013 s, p95 0.027 s, maximum 0.089 s. Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale; stronger anchor runs for that draw are being measured. Shown unranked until then.",
  "Added KnowLine 4B-Gen5.5 and KnowLine 26B-A4B-Gen2.5 (measured 23 Oct on the 12 Oct regular draw).",
  "FIRST MEASUREMENT of this serving version. SAME WEIGHTS AS Gen2 (no earlier version of this exact model is on the board): the shards, tokenizer, configuration and chat template are byte-identical to the predecessor; the new author layout reads short text inputs twice, so this measures a new serving version, not new weights. AUTHOR'S OWN SERVER on our rented hardware, bf16. NATIVE OUTPUTS: label-token logprobs, label-only softmax at temperature 1; choice argmax, Noul P(true), score expectation; header off, no rotation or calibration file. AUTHOR EXPOSURE (card, self-reported, not verified): the authors adopted read twice after checking their own evaluation sets and a fresh full run of the Decision Index 0.3 public suite; the card also reports JevBench / JevBench-hard evaluations. Training-data, Decision Index format-data and decontamination disclosures are inherited from Gen2; no overlap or contamination audit was run by us. The authors say no Jev or other decision-model outputs were used as training labels. The author's H20 timing does not transfer to this RTX PRO 6000 measurement. 10 s RULE: 15 decisions took 12.0-13.7 s (maximum 13.691 s), exactly the 15 standard very long positions, and are scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit) (median 0.0789 s, p95 0.1089 s). DETERMINISM: first 50 positions run twice, 0/50 bit-identical, 0 argmax flips, maximum probability difference 0.0077745; bit reproducibility is not established and the cause was not investigated. CONTEXT: SGLang configuration limit 262,144 tokens; no author-imposed lower cap, no truncation and 0 refusals; the longest item was 81,706 tokens. The measurement runner's internal time cap fired about 1.4 seconds before the scored loop ended. An independent review found the answer file complete: all 1,500 requests were answered, with token usage equal to the offline count. The runner's partial flag is retained in the measurement record. The answer file was lost on 26 Oct 2026 after scoring. These are the scores and confidence intervals computed on 23 Oct; the measurement cannot be scored again. Subject-topic and use-case breakdowns are unavailable. The Noul decisiveness diagnostic is unavailable because the stored summaries do not record which timed-out answers were decisive.",
  "FIRST MEASUREMENT of this serving version. SAME WEIGHTS AS Gen5 (no earlier version of this exact model is on the board): the shards, tokenizer, configuration and chat template are byte-identical to the predecessor; the new author layout reads short text inputs twice, so this measures a new serving version, not new weights. AUTHOR'S OWN SERVER on our rented hardware, online FP8. NATIVE OUTPUTS: label-token logprobs, label-only softmax at temperature 1; choice argmax, Noul P(true), score expectation; header off, no rotation or calibration file. AUTHOR EXPOSURE (card, self-reported, not verified): the authors adopted read twice after checking their own evaluation sets and a fresh full run of the Decision Index 0.3 public suite; the card also reports JevBench / JevBench-hard evaluations. Training-data, Decision Index format-data and decontamination disclosures are inherited from Gen5; no overlap or contamination audit was run by us. The authors say no Jev or other decision-model outputs were used as training labels. The author's H20 timing does not transfer to this RTX PRO 6000 measurement. 10 s RULE: 0 decisions over 10 s, maximum 2.958 s (median 0.0299 s, p95 0.0449 s). DETERMINISM: first 50 positions run twice, 6/50 bit-identical, 0 argmax flips, maximum probability difference 0.0583609; bit reproducibility is not established and the cause was not investigated. CONTEXT: SGLang configuration limit 262,144 tokens; no author-imposed lower cap, no truncation and 0 refusals; the longest item was 80,522 tokens. The answer file was lost on 26 Oct 2026 after scoring. These are the scores and confidence intervals computed on 23 Oct; the measurement cannot be scored again. Subject-topic and use-case breakdowns are unavailable.",
  "Added DecisionNode and DecisionNode Flash (hosted) on the 27 Oct regular draw, shown unranked and not priced.",
  "Hosted API by Bynn, measured directly through the vendor's synchronous endpoint. All 1,500 items were sent; 5 items over the provider's input limit were refused (HTTP 400 max_tokens_exceeded), scored as misses and kept in the denominator. The provider reports probabilities to two decimals; exact zeros on many-option items are expected, and Calibration is computed on the reported values without adjustment. Output tokens reported as 0. Measured on the 27 Oct 2026 regular draw, not yet equated to the reference scale (offset 0). This draw has no subject-topic labels yet; use-case and language cells are measured on its own draw. Latency: p50 0.256 s, p95 0.317 s, maximum 0.668 s.",
  "Hosted API by Bynn, measured directly through the vendor's synchronous endpoint. All 1,500 items were sent; 5 items over the provider's input limit were refused (HTTP 400 max_tokens_exceeded), scored as misses and kept in the denominator. The provider reports probabilities to two decimals; exact zeros on many-option items are expected, and Calibration is computed on the reported values without adjustment. Output tokens reported as 0. Measured on the 27 Oct 2026 regular draw, not yet equated to the reference scale (offset 0). This draw has no subject-topic labels yet; use-case and language cells are measured on its own draw. A first attempt stopped incomplete because of a harness resend bug on our side and is not used; the published result is the second, complete attempt. Latency: p50 0.675 s, p95 0.802 s, maximum 1.408 s.",
  "Added Lychee-27B, measured on 13 Oct 2026 on the 12 Oct regular draw as a paid priority evaluation; published at the customer's request.",
  "Measured on 13 Oct 2026 as a paid priority evaluation; published at the customer's request. The estimated cost exceeds the official cost cap, so this row is outside the default cost-capped Capability ranking.",
  "Added VisionJEV 1.0.0 (self-hosted) on the 27 Oct regular draw, shown unranked. VisionJEV 1.0.0 is a 27B model built on Qwen/Qwen3.8-27B, with a decision readout over the last-token hidden state. The repository NOTICE attributes the prompt, answer conversion and noncausal attention hook to autojev/pplx-decider. TEXT ONLY: the vision-capable model was measured without images. SERVING: the author's own shipped model and documented HTTP server, unmodified, BF16, SDPA, noncausal full attention, last-token pooling, temperature 0.986316004609301, batch size 1, no thinking, one forward pass, no truncation, quantisation, rereading or retries. Contributor code ran only on a disposable rented pod with network cut. ITEMS: all 1,500 items were sent; 5 exceeded the native 8,192-token input limit and were refused with HTTP 422, scored as misses and kept in the denominator; these carry no usage and are unbilled. 1,495 items were answered. TOKENS: output tokens reported as 0; native input usage is the observed length of input ids at the native prepare call, 562,151 input tokens on answered requests. TRAINING: training data were not inspected; overlap with public or sealed JevBench items is unchecked. DETERMINISM: 49 of 50 replayed requests were bit-identical, with 0 argmax flips (recorded maximum probability difference 0.0). WARM-UP: a public synthetic warm-up preceded the timed measurements; the first cold smoke request took about 42 seconds. LATENCY: measured loopback p50 0.152 s, p95 0.290 s, maximum 9.297 s. The share view applies the common self-hosted latency assumption, twice measured p50 plus 0.15 s, an assumption rather than a hosted-endpoint measurement. HARDWARE: one A100 80 GB GPU; weights alone occupy 48.6 GiB and smoke peak allocated memory was 50.2 GiB. LICENCE: Apache-2.0 with NOTICE preserved. This draw has no subject-topic labels yet, so the row has no topic cells; use-case and language cells are measured on its own draw. Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale; stronger anchor runs for that draw are being measured. Shown unranked until then.",
  "Added Umpire 35B-A3B (self-hosted) on the 27 Oct regular draw, shown unranked. Umpire 35B-A3B is a 35B-total, 3B-active mixture-of-experts model (qwen3_5_moe architecture), declared base ornith-ai/Ornith-1.5-35B-A3B, with final-position option-slot logits. AUTHOR CLAIM: these BF16 weights are the author's served 4-bit weights dequantized; tensor parity was not established. Only the BF16 CUDA path was measured; no MLX or other weight variant was measured. SERVING: the author's unchanged Transformers server, BF16, use_cache false, logits_to_keep 1, temperature 1, up to 255 option slots, thinking off, one forward pass, no generated text, truncation, retries or ensemble. The vLLM engine was not started. Contributor code ran only on a disposable rented pod with network cut. KERNEL PATH: flash-linear-attention and causal-conv1d were unavailable; Transformers' reference PyTorch linear-attention path is inferred from server warnings, not kernel tracing; no separate scored-run kernel trace was captured. ITEMS: all 1,500 items were sent; exactly 5 exceeded the native 16,384-token input limit, matching the preregistered count, and returned HTTP 400 context_length_exceeded, scored as misses and kept in the denominator; these carry no usage and are unbilled. 1,495 items were answered. Boundary checks answered 16,384 tokens and refused 16,385, 80,000 and 84,000 tokens with normal recovery. TOKENS: output tokens reported as 0; native input usage was 621,390 tokens on answered requests, identical to first-party offline counts with 0 usage mismatches. A one-option request returns deterministically with zero usage. TOKENIZER: exact prompt bytes, slot ids, encoded ids and usage matched over 543 public and synthetic cases, including 231 public rows. TRAINING: the card claims Open-Jev probability-distribution training and its own public test numbers; these claims were not independently verified. Training data were not inspected; overlap with public or sealed JevBench items is unchecked. DETERMINISM: 49 of 50 replayed requests were bit-identical, with 0 argmax flips (recorded maximum probability difference 0.0). WARM-UP: public pre-warm preceded the scored fresh server's timed requests; separate public smoke recorded about 22 seconds for model loading and 21 seconds for warm-up. LATENCY: measured loopback p50 0.405 s, p95 0.494 s, maximum 0.627 s; no request took 10 seconds or longer. The share view applies the common self-hosted latency assumption, twice measured p50 plus 0.15 s, an assumption rather than a hosted-endpoint measurement. HARDWARE: one A100 80 GB GPU; weights alone occupy 64.6 GiB. No peak-memory measurement is available. LICENCE: weights CC BY-NC 4.0 (non-commercial use only); serving code Apache-2.0, with MIT SemIf notice preserved. One scored run, with every pre-input gate passed before scored input. This draw has no subject-topic labels yet, so the row has no topic cells; use-case and language cells are measured on its own draw. Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale; stronger anchor runs for that draw are being measured. Shown unranked until then.",
  "Text-only corrections: removed internal process references from published notes, corrected the KnowLine and DecisionNode descriptions and the over-limit wording; no score, cost or rank changed.",
  "Cost basis for self-hosted open weights changed from base-model list prices to GPU time on a monthly-rented RTX PRO 6000 with an efficient inference engine. 210 rows have verified scored usage and a resolvable architecture; 46 additional open-weight rows fall within both official caps. Intelligence, Calibration and latency are unchanged; API costs and both caps are unchanged. Encoders, architectures outside the six calibrated Gemma/Qwen classes and rows without a verified token basis retain their old costs. Composite intervals affected by the cost change are withheld; the archived v1.7 intervals use the old cost. Old values remain available in v1.7.x. Throughput was calibrated on prompt processing (prefill); generated tokens are counted at the same rate. Three diffusion-readout rows are priced with the closest calibrated class (MoE Gemma); this is an approximation."
 ],
 "method": {
  "decision_limit_s": 10,
  "label": "v1.7",
  "rule": "latency_s > 10.000 s -> failed",
  "treatment": "share"
 },
 "n_ranked": 283,
 "not_measured": [
  {
   "addendum": null,
   "author": "unknown",
   "display": "AutoJev-27B (RTX PRO 6000)",
   "key": "autojev-27b-rtxpro6000-a2",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/denis-pplx/autojev-27b",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Clause Logic",
   "display": "Bosun v3.1 0.6B",
   "key": "bosun-v31-0.6b",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/Hanno-Labs/bosun-v3.1-0.6b",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Fastino",
   "display": "GLiNER2 (Fastino, gliner2.5-base)",
   "key": "gliner2",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/fastino-ai/GLiNER2",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Fastino",
   "display": "GLiNER2 large (Fastino)",
   "key": "gliner2-large",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/fastino/gliner2-large-v1",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Fastino",
   "display": "GLiNER2.5 multi (Fastino, 287M)",
   "key": "gliner2.5-multi",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/fastino/gliner2.5-multi-v1",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Fastino",
   "display": "GLiNER2.5 small (Fastino, 74M)",
   "key": "gliner2.5-small",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/fastino/gliner2.5-small-v1",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Logan Markewich",
   "display": "jeff (Logan Markewich, GLiFormer 400M)",
   "key": "jeff",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/logan-markewich/jeff",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "MantisShrimpdev",
   "display": "Jobe Qwen3.5-4B (frozen)",
   "key": "jobe-qwen3.5-4b",
   "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: no official score was published.",
   "repo": "https://benchmarkheaven.com/jev-models/jobe-qwen3.5-4b",
   "status": "not measured"
  },
  {
   "addendum": {
    "id": "A1",
    "label": "v1.5 roster addendum A1",
    "release": "v1.5"
   },
   "author": "unknown",
   "display": "mica-v01-4b",
   "key": "mica-v01-4b",
   "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: The frozen refusal policy stopped the run after 1,088 of 1,624 rows: 27 documented refusals were mapped to HTTP 422, then three consecutive passthrough HTTP 400 refusals triggered exit 6. The remaining 536 rows have no scores, so this system is not eligible for an official rank.",
   "repo": "https://benchmarkheaven.com/jev-models/mica-v01-4b",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Cactus Compute",
   "display": "Needle 3 (Cactus, 2-bit, local CPU)",
   "key": "needle-3",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/cactus-compute/needle",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Cactus Compute",
   "display": "Needle 3, options as tools (post-hoc adapter mode)",
   "key": "needle-3-tools",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/cactus-compute/needle",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "Igor L. / NInfer contributors",
   "display": "NInfer Qwen3.8-27B NVFP4 (T=1.5)",
   "key": "ninfer-qwen3.8-27b-t1.5",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/igorls/ninfer",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "razorback16 / Codiv",
   "display": "OpenJev (DiffusionGemma 26B-A4B NVFP4, razorback16)",
   "key": "openjev-razorback16",
   "reason": "No v1.6.0 score. Prior v1.5.6 catalogue: no official score was published.",
   "repo": "https://benchmarkheaven.com/jev-models/openjev-razorback16",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "kshetrajna12",
   "display": "reflex-27b (Qwen3.8-27B)",
   "key": "reflex-27b",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/kshetrajna12/reflex",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "unknown",
   "display": "Surogate Rune 26B-A4B v3 (RTX PRO 6000)",
   "key": "surogate-rune-26b-a4b-v3-rtxpro6000-a2",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://huggingface.co/api/models/surogate/rune-26b-a4b-GGUF",
   "status": "not measured"
  },
  {
   "addendum": null,
   "author": "wfzyx (Victor Hugo)",
   "display": "Von (wfzyx, Option-Marker 395M)",
   "key": "von-395m",
   "reason": "Not measured on the v1.6.0 pool.",
   "repo": "https://github.com/wfzyx/von",
   "status": "not measured"
  }
 ],
 "noul_method": {
  "applied": "O1S",
  "decisive_source_sha256": "159554cb4ebc276bac6e24559b3f326a47398b97479d0048647411ed64322e49"
 },
 "options": {
  "A": {
   "intelligence_floor": 50,
   "weights": {
    "calibration": 25,
    "cost": 25,
    "intelligence": 25,
    "speed": 25
   }
  },
  "B": {
   "intelligence_floor": 50,
   "weights": {
    "calibration": 20,
    "cost": 20,
    "intelligence": 40,
    "speed": 20
   }
  },
  "C": {
   "intelligence_floor": 60,
   "weights": {
    "calibration": 25,
    "cost": 25,
    "intelligence": 25,
    "speed": 25
   }
  }
 },
 "overnight": {
  "a2_note": "History: hosted APIs were first measured on API subsets (A, then A2 and A3) and equated to the self-hosted scale in v1.6.0. From v1.6.1 they answer the same full set as self-hosted systems, so no equating is applied; the earlier subsets A, A2 and A3 are retired.",
  "exposure": {
   "classifier-dev-fast": {
    "display": "classifier-dev-fast (classifier.dev)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "decision-machine-1": {
    "display": "decision-machine-1 (milliseconds.ai)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "deepseek-flash": {
    "display": "deepseek-flash (deepseek)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "fastino-gliner-2-5-decide": {
    "display": "fastino-gliner-2-5-decide (fastino)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   },
   "gemini-3.1-flash-lite": {
    "display": "gemini-3.1-flash-lite (google)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "gpt-5.6-luna": {
    "display": "gpt-5.6-luna (openai)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "gpt-6-luna": {
    "display": "gpt-6-luna (openai)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "gpt-6-luna-low": {
    "display": "gpt-6-luna-low (openai)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "instinct": {
    "display": "instinct (zoowork)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "instinct-dual-4b": {
    "display": "instinct-dual-4b (zoowork)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "jev-1.13.0": {
    "display": "jev-1.13.0 (typesafe)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   },
   "jevact": {
    "display": "jevact (jevact)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "kushal-gemma4-31b-it-autoloops": {
    "display": "kushal-gemma4-31b-it-autoloops (autoloops)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "openjev-sglang": {
    "display": "openjev-sglang (modal via modal)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "qwen3.8-27b": {
    "display": "qwen3.8-27b (chutes)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "sage-1.3.0": {
    "display": "Sage 1.3.0 (Levanto Labs)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   },
   "simplejev-qwen3.6-35b-a3b": {
    "display": "simplejev-qwen3.6-35b-a3b (featherless)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "simplejev-qwen3.8-27b": {
    "display": "simplejev-qwen3.8-27b (featherless)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "system-one-open": {
    "display": "system-one-open (modal via modal)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "vansa-3.4": {
    "display": "vansa-3.4 (vansa)",
    "draws": "v1.5 sealed-720 (retired for scoring)",
    "note": "carried: last measured on v1.5 (API-exposure rule; not due / not in tonight's scope)",
    "sealed_items_exposed": 0
   },
   "wity-1-always": {
    "display": "wity-1-always (wity via railway)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   },
   "wity-1-auto": {
    "display": "wity-1-auto (wity via railway)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   },
   "wity-1-off": {
    "display": "wity-1-off (wity via railway)",
    "draws": "S (1,200 sealed) + P (300 public)",
    "note": "full set since v1.6.1 (5 Oct 2026); earlier API-subset answers reused, only missing sealed items sent; whole v1.6.0 sealed draw retired",
    "sealed_items_exposed": 1200
   }
  },
  "notes": [
   "Scores are the official v1.6.0 scorer (score_v16.py, method option B / O1S, bootstrap B = 1,000) over complete outputs only; each scoring round is kept separately.",
   "Hosted APIs are scored on the same full item set as self-hosted systems (S 1,200 + P 300, v1.6.1) and are not equated.",
   "GPU-class deviations: reproducible recipes used the hardware listed per row, including H100 for large fast-lane decoders and RTX 6000 for the baseline; hardware differences remain in measured latency. The standard x2 + 0.15 s self-hosted adjustment is an assumption, not a hardware normalization.",
   "Jev-class caps use a fixed reference: Jev 1.13.0 as measured in v1.5 (p50 0.62 s, USD 0.0323 per 1k answers); caps = 2x (1.23 s, USD 0.0646). Jev's own v1.6 p50 is 0.24 s. Wity auto remains outside the latency cap and stays ranked in Composite A. OFF and ALWAYS are unranked variants of the AUTO main row.",
   "6 further measured candidates await a separate publication decision.",
   "Sage 1.3.0 (Levanto Labs) was measured on 5 Oct from Sandy (Helsinki), text only. Cost uses the Levanto list tariff (USD 0.05/M input, USD 10/M output; levanto.ai/pricing, read 5 Oct) over all 600 answered rows: USD 0.0766/1,000 decisions. It exceeds the frozen Jev-class cost cap and is excluded from the Capability headline.",
   "A fresh Monday Jev 1.13.0 re-check on A3 measured p50 0.239 s and equated Capability 77.5 versus the official 76.5, within the confidence interval; the official v1.6.0 Jev row is retained."
  ],
  "round": "score-v161-3 (v1.6.1)",
  "scored_utc": "2026-10-06 00:39:36 UTC"
 },
 "protocol": "jevbench::v1.7",
 "provisional": false,
 "restates": {
  "board_release": "v1.6.43",
  "feed_sha256": "f1006546d7d9614132df86542176127835b1be5eb39344bc074ac9a83f2dbb0e"
 },
 "revision": "v1.8.0",
 "revision_history": [
  {
   "date": "2026-10-05",
   "revision": "v1.6.1",
   "summary": "Hosted APIs (fastino-gliner-2-5-decide, jev-1.13.0, sage-1.3.0, wity-1, wity-1-always, wity-1-off) answer the full 1,500-item set instead of the 600-item API subset and are no longer equated; self-hosted rows unchanged; whole v1.6.0 sealed draw retired. Cost per 1,000 decisions is measured on one common item set (all items except the 23 outside the Jev reference input range) for every token-priced row."
  },
  {
   "date": "2026-10-05",
   "revision": "v1.6.0",
   "summary": "Rotating item draw (1,200 sealed + 300 public); hosted APIs on API subsets equated to the self-hosted scale; /jev-models/v1.6.0 stays available unchanged."
  }
 ],
 "roster_count": 316,
 "run_kind": "same-draw-addendum",
 "sample": {
  "open": 300,
  "published_open": 300,
  "sealed": 1200,
  "total": 1500
 },
 "sealed_share_of_intelligence": 0.5,
 "source_note": "Cost basis for self-hosted open weights changed from base-model list prices to GPU time on a monthly-rented RTX PRO 6000 with an efficient inference engine. 210 rows have verified scored usage and a resolvable architecture; 46 additional open-weight rows fall within both official caps. Intelligence, Calibration and latency are unchanged; API costs and both caps are unchanged. Encoders, architectures outside the six calibrated Gemma/Qwen classes and rows without a verified token basis retain their old costs. Composite intervals affected by the cost change are withheld; the archived v1.7 intervals use the old cost. Old values remain available in v1.7.x. Throughput was calibrated on prompt processing (prefill); generated tokens are counted at the same rate. Three diffusion-readout rows are priced with the closest calibrated class (MoE Gemma); this is an approximation.",
 "source_sha256": "9ea9f07fbd7d272028106580fcf4a2e83d9f87376df0e7d21bb2385f71823543",
 "status": "published",
 "systems": [
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "andyzhang232 (Hugging Face)",
   "axes": {
    "calibration": 91.33817262545804,
    "cost": 44.17643930820148,
    "intelligence": 71.46073788705554,
    "speed": 82.45454940142191
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.046153910328050815,
      "ece_score": 90.76921793438983,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 90.76921793438983,
      "score_before_time_limit": 90.76921793438983,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.13404825737265413,
      "brier": 0.04568060550795997,
      "ece": 0.03819846894711239,
      "n": 373,
      "score": 92.36030621057752,
      "score_before_time_limit": 92.36030621057752,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9731182795698925,
      "interval90_width": 0.23812724014336917,
      "n": 372,
      "nrps": 0.050897974834229455,
      "score": 90.88499373140681,
      "score_before_time_limit": 90.88499373140681,
      "time_limit_failed_share": 0,
      "top_ece": 0.06570107526881724
     }
    },
    "score": 91.33817262545804
   },
   "capability": 81.3994552562568,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 8583.03975179 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=native). The row's own scored cost basis: 1,530,069 input + 0 output tokens over 1,485 decisions = 1030.34949495 tokens/decision. GPU-time cost USD 0.0725747179002238/1,000; old list-price cost USD 0.0927314545454545/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0725747179002238/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.09180413999999999
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0725747179002238
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "AJev lm6q (Gemma 4 31B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h. Stock vLLM 0.30.0 and our shim (mounted read-only, sha re-checked on the node) in ONE docker container with network none (the cut proven from inside at start and again by docker exec from outside: loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, only the output directory writable, env -i with every credential-named variable empty, HF offline flags, the sealed input not present until the smoke and the time gate had passed (input sha256 dfd36720... checked on the node and inside the container); one request at a time, timed over loopback on the VM. Measured 22 Oct 2026 (VM reserved 09:44Z, scored run 09:55-10:01Z, released 10:01Z, about USD 0.76) on the frozen v1.6-regular-20261012 self-hosted input; node shredded and VM removed afterwards (teardown confirmed).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 73.10537789504917,
    "I_sealed": 69.81609787906191,
    "base": 71.46073788705554,
    "excess": -6.550246754368452,
    "gap": 3.2892800159872593,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 78.86591280029805,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 75.46398238439761,
       "judge": 72.31377719182598,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 75.82125603864733,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 64.62896484620214,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 75.51666666666668,
       "hard": 76.90657407407407,
       "judge": 58.80593220338983,
       "standard": 43.36444444444443
      }
     },
     "sealed|choice": {
      "cc": 84.1552380959759,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 81.78559037816582,
       "judge": 84.1332027424094,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 64.82841399465178,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 49.09090909090909,
       "judge": 72.30769230769232,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 60.46464154655807,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 79.05447368421052,
       "hard": 61.654957746478914,
       "judge": 55.02755208333334,
       "standard": 56.944727272727256
      }
     }
    }
   },
   "jevbench_score": 52.29743457752002,
   "key": "ajev-gemma4-31b-lm6q",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (model card metadata; no LICENSE file in the repository; base google/gemma-4-31B-it: apache-2.0 in its card metadata with a link to the Gemma 4 licence), public, not gated; the card warns that some training data carries non-commercial or attribution terms",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "andyzhang232/ajev-gemma4-31b-lm6q@bdcfb05db02f48729f5f5a0b6b1c8cf6d6406d06",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9814241486068112,
    "decisive": 323,
    "decisive_rate": 0.8659517426273459,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 120,
   "ranked": true,
   "ranks": {
    "A": 120,
    "B": 114,
    "C": 99,
    "capability": 27
   },
   "repo": "https://huggingface.co/andyzhang232/ajev-gemma4-31b-lm6q",
   "scores": {
    "A": 52.29743457752002,
    "B": 52.95942267288998,
    "C": 52.29743457752002
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5768386019999753,
    "p50_s_raw": 0.21341930099998763,
    "p95_s_adjusted": 0.9851236474000756,
    "p95_s_raw": 0.4175618237000378
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "NO AUTHOR CODE WAS EXECUTED OR IMPORTED. The Hub repository holds a rank-64 LoRA adapter (safetensors + configs only, no .py, no auto_map): andyzhang232/ajev-gemma4-31b-lm6q at bdcfb05db02f48729f5f5a0b6b1c8cf6d6406d06 (Apache-2.0 per card, not merged), applied directly by stock vLLM 0.30.0 (image vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90, no derived image, no overlay wheels) on google/gemma-4-31B-it at 842da3794eaa0b77d5f08bae87a17459d91ff475. The request/response layer is OUR first-party shim ajev_systemone.py (sha256 e849fcdbaf0740f3a22b61108053572b3a76d3b35030aa39ed6b00f560bcbb9a), a rule-by-rule port of the author's published inference package (cmzy/ajev-infer at e2e723aba95a1f15d51966f43b53430372f49a4d, read as text only; the unit tests compare the author's constants by ast, never exec): request normalisation, prompt v3 (v2 layout with system message for choice questions of up to 26 options, v1 layout otherwise, user text repeated once when it is at most 12,000 characters), Gemma-4 chat template with thinking off, no truncation, lettered options A-Z then tokenizer-verified two-letter codes up to 255, readout over both single-token forms of each label combined by logsumexp, softmax over labels at T=1 (the adapter stores no temperatures). One /v1/completions call per decision (max_tokens 1, temperature 0, allowed_token_ids = all label forms, logprobs = their count). Server flags: --enable-lora --max-lora-rank 64 --max-loras 1 --lora-modules ajev-lm6q=<adapter snapshot> --dtype bfloat16 --max-model-len 40960 --gpu-memory-utilization 0.88 --enable-prefix-caching --max-logprobs 512 --logprobs-mode processed_logprobs --generation-config vllm --max-num-seqs 16 --max-num-batched-tokens 8192. Review: independent reviews of the shim and the runner preceded the run; their findings were fixed and re-reviewed, and the shim hash above is the one that ran. Weights: both snapshots downloaded revision-pinned and every file sha256/inventory-verified against MODEL_MANIFEST.json (sha256 dade7dfa4333fd15ef79c5a1682e4c3a3a6fd79a45035b251d6704a0492ad086): base 12 files, 62,578,686,256 bytes; adapter 5 files, 1,959,002,147 bytes; verdict OK for both. Smoke before any item: a separate smoke-only Verda pod (passed on attempt 12) and an in-container synthetic probe on the scored VM; no test item was ever used in a smoke. FIRST MEASUREMENT (found by our own intake: Decision Index 0.4.1 entry). SERVED BY OUR SHIM, NOT BY THE AUTHOR'S SERVER: the row measures the author's documented vLLM direct-LoRA serving path as re-implemented by us (stock vLLM 0.30.0 + our shim), not necessarily the configuration behind the author's self-reported 65.08 (the card says only that it was self-run with the official kit); there is no numeric parity check against the author's code (running it is not allowed here), only a rule-by-rule port plus tests. DISCLOSED DEVIATIONS from the author's server: D1 score probabilities are a dict keyed by level (the adapter needs a dict; the author returns a list), rounded to 4 decimals like the author; D2 a missing label token is an error instead of the author's -1e4 fill (cannot change an answer here, all label ids are requested); D3 requests go through vLLM's HTTP /v1/completions route instead of in-process generation; D4 an upstream 'maximum context length' 400 would be mapped to 422 (audited separately; never happened); D5 batch parameters --max-num-batched-tokens 8192 --max-num-seqs 16 are our pins, because the author's engine most likely runs 2,496 / 128 (source-derived, not verified live) and an explicit 2,048 is rejected by vLLM 0.30.0 for Gemma-4; this changes chunk boundaries and batch shape only; the context limit 40,960 follows the author's inference README for this 31B submission (the author's server default is 131,072); output_tokens is reported as 0 like the author. OPEN PARITY RISK (not tested): the adapter has v_proj LoRA on 50 of 60 layers and none on the 10 global layers where K and V share one projection; vLLM's fused qkv_proj loads K into both slots, so a K delta may not reach V as it would under Hugging Face PEFT. The smoke proved the adapter is really applied (LoRA-vs-base total variation up to 0.95 on a choice case) and that our readout equals an independent recomputation from raw upstream logprobs; during the smoke attempts the probe-only tolerances were loosened by us (a looser bound for cases with more than 26 options, 1e-6 to 1e-3, then one gross-error bound of 0.03 / TV 0.10) after float-level noise, the shim stayed byte-identical and no item was involved. CONTEXT: our shim answers HTTP 422 above 40,960 tokens (no truncation); the 15 items of 76,562-81,737 tokens were refused exactly as pre-registered by id and position and are scored wrong; the largest answered item is 3,595 tokens. DETERMINISM: over the first 50 positions, two passes, 6/50 were bit-identical, 0 argmax flips, max probability difference 0.029 (vLLM batching and prefix-cache bf16 noise); the probabilities need not equal the author's or the DI lab's. TRAINING DATA, the author's own statement: the card names no training datasets and warns that some training data carries non-commercial or attribution terms ('check them yourself before commercial use'; not a conclusion about the weights); the training-base revision is undeclared (PEFT revision null), so our base pin is a serving pin, not provenance. The author states it ran the full public Decision Index 0.3 suite (140,620 requests) with the official kit and compared adapters on it; the author's code comments describe benchmark-guided prompt-format changes (our static triage: development exposure, no item-specific rule found); no JevBench use or denial is declared; not independently verified. SELF-REPORTED, unverified: public Decision Index 65.08 (the DI kit rescore is 65.46; the DI lab's Decision Index 0.4.1 full score is 67.57, decision agreement with the submitted rows 99.80% on 831 rows against a 98.5% bar), latency median 139 ms on one RTX PRO 6000. 10 s RULE: max measured decision latency 0.81 s; 0 decisions over 10 s.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 87.43555555555558,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 70.64000000000004
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 87.43555555555558
      },
      "calibration_gen": {
       "accuracy": 0.801980198019802,
       "cc": 68.26435357016614,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 60.00000000000001,
        "score": 70.24374999999998
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 68.26435357016614
      },
      "composite_gen": {
       "accuracy": 0.9306930693069307,
       "cc": 88.10313540870452,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 74.96884615384616
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 88.10313540870452
      },
      "fairness": {
       "accuracy": 1,
       "cc": 98.24777777777778,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 94.74333333333334
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 98.24777777777778
      },
      "jaggedness": {
       "accuracy": 0.7888888888888889,
       "cc": 58.138797836504914,
       "cc_by_type": {
        "choice": 76.90086621751685,
        "noul": 35.13513513513513,
        "score": 62.380392156862754
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 58.138797836504914
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 83.77958729004955,
       "cc_by_type": {
        "choice": 95.24522321035484,
        "noul": 95.8762886597938,
        "score": 60.21725
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.77958729004955
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.8148148148148148,
       "cc": 65.07543446244478,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 75,
        "score": 44.96857142857144
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 65.07543446244478
      },
      "multilingual": {
       "accuracy": 0.7818181818181819,
       "cc": 62.92410190836234,
       "cc_by_type": {
        "choice": 78.4805999347897,
        "noul": 47.22222222222223,
        "score": 63.0694835680751
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 62.92410190836234
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 82.18677777777778,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 80,
        "score": 73.227
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 82.18677777777778
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 87.12279086779087,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 73.38038461538463
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 87.12279086779087
      },
      "usecase": {
       "accuracy": 0.7181818181818181,
       "cc": 58.217503006535956,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 45.83333333333333,
        "score": 55.77996
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 58.217503006535956
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 58.70851037851037,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100,
        "score": 14.220769230769214
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 58.70851037851037
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 82.38133333333333,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 47.143999999999984
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 82.38133333333333
      },
      "de": {
       "accuracy": 0.875,
       "cc": 83.51293446197256,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 74.43260869565218
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.51293446197256
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 65.27459342023587,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 96.00999999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 65.27459342023587
      },
      "en": {
       "accuracy": 0.8496959165942659,
       "cc": 73.91733013139692,
       "cc_by_type": {
        "choice": 84.8427527123174,
        "noul": 72.66187050359711,
        "score": 64.24736717827624
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 73.91733013139692
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 68.17863608738764,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 24.111764705882344
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 68.17863608738764
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 65.679927140255,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 96.49333333333333
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.679927140255
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 94.5440350877193,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 83.6321052631579
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 94.5440350877193
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 77.16916666666667,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 75,
        "score": 56.50750000000001
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 77.16916666666667
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 63.860603496269896,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 5.6114285714285455
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 63.860603496269896
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 43.283918453799316,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 60.00000000000001,
        "score": 27.04000000000001
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 43.283918453799316
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 10.035661375661372,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 74.55142857142856
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 10.035661375661372
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.2838888888889,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 95.51833333333335
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.2838888888889
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -3.348675324675322,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 15.927999999999997
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 67.13170542635659,
       "cc_by_type": {
        "choice": 100,
        "noul": 39.99999999999999,
        "score": 61.395116279069775
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 67.13170542635659
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 76.00928311187991,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 59.84870967741933
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 76.00928311187991
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 58.087695035461,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 39.79500000000002
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 58.087695035461
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 63.50320652173914,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50,
        "score": 62.24875000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 63.50320652173914
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.80564516129031,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 50.67499999999998
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.80564516129031
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 53.97777777777778,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 28.6
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 53.97777777777778
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8733333333333333,
       "cc": 83.63918176773377,
       "cc_by_type": {
        "choice": 83.63918176773377
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 83.63918176773377
      },
      "noul": {
       "accuracy": 0.8453333333333334,
       "cc": 69.06666666666668,
       "cc_by_type": {
        "noul": 69.06666666666668
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 69.06666666666668
      },
      "score": {
       "accuracy": 0.7573333333333333,
       "cc": 62.66353790613719,
       "cc_by_type": {
        "score": 62.66353790613719
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 62.66353790613719
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 16.118627450980387,
       "cc_by_type": {
        "choice": 52,
        "noul": -25,
        "score": 21.35588235294116
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 16.118627450980387
      },
      "code_linting": {
       "accuracy": 0.8333333333333334,
       "cc": 84.18102564102564,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 52.54307692307694
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 84.18102564102564
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 60.77650406504063,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100,
        "score": 31.10999999999997
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 60.77650406504063
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -27.201396198830412,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -11.42875000000001
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 59.12927536231884,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 73.03999999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.12927536231884
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 33.06406708595387,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 66.66666666666667,
        "score": -17.160000000000018
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 33.06406708595387
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 33.46719026071966,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -14.28571428571429,
        "score": 34.29512820512819
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 33.46719026071966
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 62.090661515931934,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 50,
        "score": 67.63086956521738
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 62.090661515931934
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 46.62185185185185,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 60.00000000000001,
        "score": 21.53222222222221
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 46.62185185185185
      },
      "knowledge_graphs": {
       "accuracy": 0.7727272727272727,
       "cc": 66.72214273354531,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 46.347727272727276
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 66.72214273354531
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 68.96410774410775,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 29.11454545454546
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 68.96410774410775
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 67.53420698924731,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 67.11874999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 67.53420698924731
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 60.68493211061349,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 73.57111111111111
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 60.68493211061349
      },
      "model_routing": {
       "accuracy": 0.8940092165898618,
       "cc": 79.85720542478356,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 100,
        "score": 48.92226562500001
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.85720542478356
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 24.02750291671371,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 83.59882352941176
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 24.02750291671371
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 61.275080470442596,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 67.20500000000003
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 61.275080470442596
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 24.474419974886505,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0,
        "score": 17.758235294117654
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 24.474419974886505
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 29.065942028985507,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": -33.333333333333336,
        "score": 49.516666666666666
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 29.065942028985507
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.73052631578946,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.19157894736841
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.73052631578946
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.499122489912,
      91.4938757484774
     ],
     "I": [
      66.4121378530749,
      75.44492915004695
     ],
     "capability": [
      77.54868244351084,
      82.87442403920699
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.33817262545804,
     "capability": 81.3994552562568,
     "intelligence": 71.46073788705554
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8733333333333333,
      "calibration": 90.76921793438983,
      "cc": 83.63918176773377,
      "cc_by_type": {
       "choice": 83.63918176773377
      },
      "cc_tiered": 81.51057544813698,
      "cc_tiered_open": 78.86591280029805,
      "cc_tiered_sealed": 84.1552380959759,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.63918176773377
     },
     "noul": {
      "accuracy": 0.8453333333333334,
      "calibration": 92.36030621057752,
      "cc": 69.06666666666668,
      "cc_by_type": {
       "noul": 69.06666666666668
      },
      "cc_tiered": 70.32483501664956,
      "cc_tiered_open": 75.82125603864733,
      "cc_tiered_sealed": 64.82841399465178,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 69.06666666666668
     },
     "score": {
      "accuracy": 0.7573333333333333,
      "calibration": 90.88499373140681,
      "cc": 62.66353790613719,
      "cc_by_type": {
       "score": 62.66353790613719
      },
      "cc_tiered": 62.546803196380104,
      "cc_tiered_open": 64.62896484620214,
      "cc_tiered_sealed": 60.46464154655807,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.66353790613719
     }
    },
    "raw_on_A": {
     "calibration": 88.32281914978606,
     "capability": 79.47232648522377,
     "intelligence": 70.6218338206615
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 48.03037255363918,
    "Emphasis on Accuracy 60:20:20": 50.85791535700579,
    "Emphasis on Cost 20:20:60": 41.50877562524695,
    "Emphasis on Speed 20:60:20": 53.457172777591474,
    "Intelligence only": 55.7839038796579
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ALERIA-ML (GitHub; Aleria AI)",
   "axes": {
    "calibration": 85.70385668097765,
    "cost": 41.14441351959124,
    "intelligence": 68.01802916815697,
    "speed": 86.37984483839423
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06242195896437067,
      "ece_score": 87.51560820712587,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 86.34873343103085,
      "score_before_time_limit": 87.51560820712587,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.06933333333333336,
      "brier": 0.07996263937935019,
      "ece": 0.0693347575324291,
      "n": 375,
      "score": 85.6736722348821,
      "score_before_time_limit": 86.13304849351418,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9653333333333334,
      "interval90_width": 0.3844444444444446,
      "n": 375,
      "nrps": 0.07381293040469014,
      "score": 85.08916437702001,
      "score_before_time_limit": 85.77536731554437,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.10533986164221125
     }
    },
    "score": 85.70385668097765
   },
   "capability": 76.86094292456731,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,273,730 input + 0 output tokens over 1,485 decisions = 1531.13131313 tokens/decision. GPU-time cost USD 0.0915910172859006/1,000; old list-price cost USD 0.137801818181818/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0915910172859006/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.34703663999999995
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.09159101728590062
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Aleria Judge 1.1.1 (ALERIA-ML, inference technique on stock Gemma 4 31B, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.535/h. The author's own `python -m aleria_judge` from the pinned commit on 127.0.0.1:30104 inside a network-none Docker container with the cut proven from inside before the input was read (loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, read-only weights and task input, empty credential placeholders (checked by name in image and container), HF offline flags, trace off, one request at a time. Timed over loopback on the VM. Measured 22 Oct 2026 (VM created 07:57Z, removed 08:23Z) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720..., checked locally, on the node and inside the container); determinism 50/50 bit-identical; node input shredded and VM and volume destroyed afterwards (teardown confirmed).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 70.49470287748886,
    "I_sealed": 65.5413554588251,
    "base": 68.01802916815697,
    "excess": -4.886179351691943,
    "gap": 4.953347418663768,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 79.62086718847044,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 77.35136835482857,
       "judge": 72.31377719182598,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 74.86473429951691,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 65.21739130434783,
       "judge": 70,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 56.99850714447922,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.51168140265938,
       "hard": 60.80173971760845,
       "judge": 58.569917341521595,
       "standard": 41.77833957356712
      }
     },
     "sealed|choice": {
      "cc": 79.4026656127611,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 76.06106163987509,
       "judge": 79.72575905974536,
       "standard": 80.06749630879561
      }
     },
     "sealed|noul": {
      "cc": 70.14222113450113,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 89.65517241379311,
       "hard": 63.63636363636365,
       "judge": 72.30769230769232,
       "standard": 70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 47.07917962921306,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 68.54236024220923,
       "hard": 45.06526528589918,
       "judge": 41.97881382833055,
       "standard": 48.02596671066647
      }
     }
    }
   },
   "jevbench_score": 43.51029656636035,
   "key": "aleria-judge-1.1.1",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (code repo LICENSE, NOTICE and pyproject; the weights are stock google/gemma-4-31B-it, model card apache-2.0, no LICENSE file in the tree; the NVIDIA TensorRT-LLM base image carries NVIDIA's product terms), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "ALERIA-ML/aleria-judge@ed2fcb1ac7bef5ef5e392a971dfe4f39ee53080d",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9312320916905444,
    "decisive": 349,
    "decisive_rate": 0.935656836461126,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 137,
   "ranked": true,
   "ranks": {
    "A": 137,
    "B": 124,
    "C": 112,
    "capability": 55
   },
   "repo": "https://github.com/ALERIA-ML/aleria-judge",
   "scores": {
    "A": 43.51029656636035,
    "B": 43.99705055687304,
    "C": 43.51029656636035
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.41476367199984454,
    "p50_s_raw": 0.13238183599992226,
    "p95_s_adjusted": 0.554900102999988,
    "p95_s_raw": 0.202450051499994
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The author's own `aleria_judge` package (ALERIA-ML/aleria-judge, tag v1.1.1 = commit ed2fcb1ac7bef5ef5e392a971dfe4f39ee53080d, tree e7caf1e6182e22ed58e90e749a750a1bbc4e135b, Apache-2.0, 62 files, 60 text files hash-verified by git blob oid; `main` HEAD is the same commit), unmodified: our harness image adds a second layer on top of the author's own Dockerfile (Dockerfile + aleria_judge/**, built `--network none --pull=false`; its `RUN verify_runtime()` step checks the TensorRT-LLM sources), started as the author's `python -m aleria_judge` (the Dockerfile CMD) with the author's own defaults: `JUDGE_ENGINE=lean` (own Triton/FlashInfer token-tree forward, BF16), effort `high` (no options.effort sent), vision tower loaded (`JUDGE_VISION=true`, 560 soft tokens), `JUDGE_CPU_THREADS=8`, plus `JUDGE_MODEL_PATH=/model`, `JUDGE_HOST=127.0.0.1`, `JUDGE_TRACE=false` and `JUDGE_CONTEXT_LENGTH=131072` (the author's submitted 1.1.1 default for the lean engine, config.py:21-24; /health reported differs_from_evaluated {} before and after the sealed run). The author's compose file was not used. Static source review PASS WITH FINDINGS, no blockers (no subprocess, eval/exec, pickle, torch.load, sockets or HTTP clients in the author package; no trust_remote_code; no rule keyed to item text or benchmark names; item text can leave the process only through /v1/trace, which was switched off). DISCLOSED CODE PATTERNS, cleared by the benchmark owner (21 Oct 2026; author code on a rented Verda pod only, network none): (1) the author's default start downloads the 62.6 GB checkpoint from the Hub (aleria_judge/checkpoint.py:98, probe at :90; the compose file makes this the default) - never reached, because JUDGE_MODEL_PATH=/model returns at checkpoint.py:80-83 with a pre-staged, sha256-verified checkpoint, HF offline flags, empty credentials and a network-none container; (2) the author's compose file uses network_mode: host and passes HF_TOKEN through (docker-compose.yml:28,50) - not used; (3) FlashInfer/Triton JIT-compile kernels at first use (lean_core.py:342, @triton.jit) and the NVIDIA image is not source-audited - run with tmpfs scratch inside the network-none container, after a synthetic smoke in the same container shape. Weights: google/gemma-4-31B-it at 842da3794eaa0b77d5f08bae87a17459d91ff475 (stock, not gated), the 9 serving files (62,578,656,403 bytes) fetched by us and sha256-verified on the host against the pinned revision, mounted read-only at /model (a mount deviation accepted by the benchmark owner) and re-verified inside the container before any item was read. Image: NVIDIA nvcr.io/nvidia/tensorrt-llm/release@sha256:124da3f10b73afbf4242f5c68fb04b48736c61fbe662e715202371fd7f046f43 (tag 1.3.0rc29, pulled by digest on the pod before the container network was cut); run image id sha256:f309f6a9dfc5ee6b20c9601614952fd00cbcbcc7456051e902a4aaadeac74fb9. In-container code check: 29 author files identical to the pinned manifest (0 failures); the stock base image's own tensorrt_llm/ tree (1,120 extra files, 153 of them .py, tree sha256 5551af64...98ed5) is admitted only by that pinned tree hash plus a package-identity check, both OK (this allowance was added by us after two pre-sealed attempts stopped at the code check because the stock image ships that source tree; no item was involved). FIRST MEASUREMENT (found by our own intake: open submission, Decision Index 0.4.1 row 13; it replaces the earlier board entry morj-1.0.1 of the same project, formerly named MORJ; neither was on JevBench before). INFERENCE TECHNIQUE ON STOCK WEIGHTS: no fine-tuning and no generated text; the author reads the model's next-token distribution over option labels for several rotations of the option order and combines them (README). SEVERAL PROMPTS PER QUESTION: see cost_basis; the native usage is the sum over all scored branches and was verified item by item. AUTHOR-DEFAULT CONTEXT LIMIT 131,072 (the author's submitted 1.1.1 default for the lean engine; the author's public Decision Index run used 65,536 and states that requests within 65,536 get identical answers at the higher limit, checked on 200 index requests): the server answers HTTP 422 above the limit, never a truncation; no status proxy was used. On this draw there were 0 refusals and 1,500 HTTP 200 answers. The 15 items whose longest branch is 76k-82k tokens (76,595-81,839) were ANSWERED, but each took 24.9-27.8 s, so under the JevBench 10 s rule they count as failed decisions; at the author's public-run limit of 65,536 the same 15 items would have been refused, so either limit scores them as failed. Every other decision took 0.384 s or less (median 0.13 s). BF16, effort high (the author's default) and the vision tower loaded although the items are text only, as in the author's compose defaults. SERIAL: we sent strictly one request at a time; determinism over the first 50 positions was bit-identical (50/50, 0 argmax flips). The author states that last-bit probability differences are possible on other hardware. METHOD AND TUNING, the author's own statements (docs/method.md, unverified): the Decision Index was also the author's development benchmark and the serving defaults were selected on index runs; the option-count temperature curve (for questions with 3+ options) was fitted on 512 examples from the training splits of BANKING77, CLINC150, WANLI and ARC-Challenge, of which BANKING77, CLINC150 and ARC are also sources of index benchmarks; the author reports that one fit example duplicates a CLINC150 test item up to punctuation and 7 more differ from one by a word or two, and that refitting without them gives the same constants or moves one grid step. We did not check this independently. The prompt layout comes from Mapika/decider, the question list from wcole3/jev-brain. SELF-REPORTED, unverified: public Decision Index 0.3 score 63.43 on H100 PCIe 80 GB and 63.46 on RTX PRO 6000, median / p80 / mean latency 119 / 403 / 320 ms on RTX PRO 6000 (README); the Decision Index lab reports 99.90% decision agreement with the submitted rows on 831 rows (bar 98.5%). HARDWARE: RTX PRO 6000 96 GB, the card the author validated on; latency reflects that card. 10 s RULE: 15 of 1,500 decisions exceeded 10 s (max 27.79 s); no other decision was near the limit.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 76.34111056721059,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 45.689998368298454
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 76.34111056721059
      },
      "calibration_gen": {
       "accuracy": 0.8712871287128713,
       "cc": 77.69535970051153,
       "cc_by_type": {
        "choice": 71.36797454931072,
        "noul": 92,
        "score": 69.71810455222389
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 77.69535970051153
      },
      "composite_gen": {
       "accuracy": 0.8514851485148515,
       "cc": 77.47093029762064,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 85.18518518518519,
        "score": 63.48776830930274
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 77.47093029762064
      },
      "fairness": {
       "accuracy": 0.8958333333333334,
       "cc": 88.40516683287034,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": 84.26311954623004
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 88.40516683287034
      },
      "jaggedness": {
       "accuracy": 0.6444444444444445,
       "cc": 41.748078200203885,
       "cc_by_type": {
        "choice": 68.23869104908566,
        "noul": 35.13513513513513,
        "score": 21.870408416390863
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 41.748078200203885
      },
      "llm_router": {
       "accuracy": 0.9137380191693291,
       "cc": 82.52291836464019,
       "cc_by_type": {
        "choice": 92.07537201725808,
        "noul": 93.81443298969072,
        "score": 61.67895008697178
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 82.52291836464019
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7037037037037037,
       "cc": 41.005561940426084,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 25,
        "score": 22.75895386251534
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 41.005561940426084
      },
      "multilingual": {
       "accuracy": 0.7563636363636363,
       "cc": 62.40099059114015,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": 61.111111111111114,
        "score": 54.45834256645017
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 62.40099059114015
      },
      "negation_pair_gen": {
       "accuracy": 0.8604651162790697,
       "cc": 67.74274557620764,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 60.00000000000001,
        "score": 49.894903395289546
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 67.74274557620764
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 84.94291126482074,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 69.84374880947725
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 84.94291126482074
      },
      "usecase": {
       "accuracy": 0.759090909090909,
       "cc": 64.88213894006674,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 70.83333333333333,
        "score": 44.646416820200216
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 64.88213894006674
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 27.79711937467592,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50,
        "score": -47.561022828353195
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 27.79711937467592
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 82.047283199588,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 66.87917678770407
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 82.047283199588
      },
      "de": {
       "accuracy": 0.875,
       "cc": 84.87936515069799,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 78.5319007618285
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 84.87936515069799
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 53.368376750417035,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 93.6246833238768
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 53.368376750417035
      },
      "en": {
       "accuracy": 0.8184187662901824,
       "cc": 69.53360743701114,
       "cc_by_type": {
        "choice": 81.54769895412551,
        "noul": 76.25899280575538,
        "score": 50.79413055115255
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 69.53360743701114
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 66.73821861479911,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 19.79051228811676
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.73821861479911
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 64.81788141933258,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 93.90719617056605
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 64.81788141933258
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 80.95313634625022,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 64.03587962698593
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 80.95313634625022
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 67.89002872687081,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 50,
        "score": 53.67008618061242
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 67.89002872687081
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 57.83441482601925,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -12.467137439323395
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 57.83441482601925
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 49.453900162875755,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 60.00000000000001,
        "score": 16.95582280792829
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 49.453900162875755
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 77.85294894387718,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100,
        "score": 78.003291276076
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 77.85294894387718
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.37352154231557,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 92.78723129361339
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.37352154231557
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 36.12208525541918,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 26.980974380976164
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 36.12208525541918
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 65.24982382273507,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 60.00000000000001,
        "score": 50.984342394993185
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 65.24982382273507
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 68.56571063038,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 48.1249456801795
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 68.56571063038
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 45.97867858809504,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 28.999865551519164
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 45.97867858809504
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 63.77894721198077,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100,
        "score": 34.815102505507525
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 63.77894721198077
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 73.10560716642341,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 51.57488601539926
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 73.10560716642341
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 62.37911574389773,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 53.80401389835985
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 62.37911574389773
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8453333333333334,
       "cc": 80.02257984270653,
       "cc_by_type": {
        "choice": 80.02257984270653
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 80.02257984270653
      },
      "noul": {
       "accuracy": 0.8666666666666667,
       "cc": 73.33333333333334,
       "cc_by_type": {
        "noul": 73.33333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 73.33333333333334
      },
      "score": {
       "accuracy": 0.672,
       "cc": 50.375231807687534,
       "cc_by_type": {
        "score": 50.375231807687534
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 50.375231807687534
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.8181818181818182,
       "cc": 63.70973568583323,
       "cc_by_type": {
        "choice": 84,
        "noul": 75,
        "score": 32.1292070574997
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 63.70973568583323
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 73.25072028446563,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 19.752160853396884
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 73.25072028446563
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 59.529818099139305,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 11.109779500669948
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 59.529818099139305
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 25.070612463705256,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": 12.053942654273664
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 25.070612463705256
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 57.83185504321008,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 69.14773904267372
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 57.83185504321008
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": 16.455332023352472,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": -33.65287185447088
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 16.455332023352472
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 48.0998002228388,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 21.050100948628447
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 48.0998002228388
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 22.939882495237857,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -50,
        "score": 50.178532503135166
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 22.939882495237857
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 31.04613908628008,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": -4.361582741159764
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 31.04613908628008
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 73.53099270250745,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100,
        "score": 46.28890958503842
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 73.53099270250745
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 62.74756127877338,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 10.464906058542367
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 62.74756127877338
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 39.69115230243343,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 50.25625260622504
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 39.69115230243343
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 72.37465401331396,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 57.12396203994186
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 72.37465401331396
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 82.39917006452004,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100,
        "score": 53.04166603771596
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 82.39917006452004
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 55.11651371368234,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0,
        "score": 76.86585592031764
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 55.11651371368234
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.88731244243987,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 64.75478783799744
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.88731244243987
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 29.047727091875924,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": 16.699831521933174
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 29.047727091875924
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 45.371680903544984,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 46.259970246866835
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 45.371680903544984
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 97.62489117390125,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 92.87467352170374
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.62489117390125
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.5578208281358,
      87.22858339554696
     ],
     "I": [
      62.68314091383449,
      72.03922327034007
     ],
     "capability": [
      72.83738772532513,
      78.93901389771851
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.70385668097765,
     "capability": 76.86094292456731,
     "intelligence": 68.01802916815697
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8453333333333334,
      "calibration": 86.34873343103085,
      "cc": 80.02257984270653,
      "cc_by_type": {
       "choice": 80.02257984270653
      },
      "cc_tiered": 79.51176640061577,
      "cc_tiered_open": 79.62086718847044,
      "cc_tiered_sealed": 79.4026656127611,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 80.02257984270653
     },
     "noul": {
      "accuracy": 0.8666666666666667,
      "calibration": 85.6736722348821,
      "cc": 73.33333333333334,
      "cc_by_type": {
       "noul": 73.33333333333334
      },
      "cc_tiered": 72.50347771700902,
      "cc_tiered_open": 74.86473429951691,
      "cc_tiered_sealed": 70.14222113450113,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 73.33333333333334
     },
     "score": {
      "accuracy": 0.672,
      "calibration": 85.08916437702001,
      "cc": 50.375231807687534,
      "cc_by_type": {
       "score": 50.375231807687534
      },
      "cc_tiered": 52.03884338684614,
      "cc_tiered_open": 56.99850714447922,
      "cc_tiered_sealed": 47.07917962921306,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.375231807687534
     }
    },
    "raw_on_A": {
     "calibration": 85.96699197533792,
     "capability": 76.92406180470816,
     "intelligence": 67.8811316340784
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 40.16012191528895,
    "Emphasis on Accuracy 60:20:20": 42.3282549224135,
    "Emphasis on Cost 20:20:60": 34.13282064751902,
    "Emphasis on Speed 20:60:20": 45.91627101254835,
    "Intelligence only": 46.05807554026913
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Amazon / AWS (Hugging Face; serving code strands-labs/strands-decider on GitHub)",
   "axes": {
    "calibration": 86.70498962717834,
    "cost": 71.89605102495447,
    "intelligence": 46.72626896403243,
    "speed": 87.9627675790118
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05406852459016399,
      "ece_score": 89.1862950819672,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 89.1862950819672,
      "score_before_time_limit": 89.1862950819672,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2788203753351206,
      "brier": 0.12438675163538872,
      "ece": 0.08593619302949061,
      "n": 373,
      "score": 82.81276139410187,
      "score_before_time_limit": 82.81276139410187,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.978494623655914,
      "interval90_width": 0.6769713261648744,
      "n": 372,
      "nrps": 0.09662368737455203,
      "score": 88.11591240546593,
      "score_before_time_limit": 88.11591240546593,
      "time_limit_failed_share": 0,
      "top_ece": 0.07052903225806459
     }
    },
    "score": 86.70498962717834
   },
   "capability": 66.71562929560538,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 734,625 input + 1,485 output tokens over 1,485 decisions = 495.696969697 tokens/decision. GPU-time cost USD 0.00864566292117681/1,000; old list-price cost USD 0.0448227272727273/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00864566292117681/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.044374500000000004
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.008645662921176805
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Amazon Strands Decider 26B-A4B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.1",
    "results_sha256": "c4144b4d45aa47ca82920669535af915a1926d743bd60bf007ad958a82a437dd",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Author's server on evaluator-rented Verda RTX PRO 6000 Blackwell 96 GB, FIN-03 Finland, EU, PAYG non-spot. Measured 24 October 2026, 17:59-18:10 UTC, serial loopback requests inside a network-isolated read-only container with pinned local checkpoint/base cache, offline flags and empty credentials. All 1500 requests completed; shred and exact-instance teardown verified. Approximately USD 0.48 GPU rental is separate from the estimated inference tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 52.3716274346206,
    "I_sealed": 41.08091049344425,
    "base": 46.72626896403243,
    "excess": 1.451190170820638,
    "gap": 11.290716941176349,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.08897277377838,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 67.9144385026738,
       "judge": 68.35860250494397,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 46.2415458937198,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 47.82608695652173,
       "judge": 19.999999999999996,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 40.784363636363636,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 38.09696969696971,
       "hard": 43.399444444444434,
       "judge": 49.34,
       "standard": 24.064444444444444
      }
     },
     "sealed|choice": {
      "cc": 70.97287460698891,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 80.94027954256671,
       "hard": 65.13241586677461,
       "judge": 74.43682664054849,
       "standard": 72.47416156928918
      }
     },
     "sealed|noul": {
      "cc": 21.675723144585735,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 48.27586206896552,
       "hard": 10.909090909090914,
       "judge": 10.769230769230775,
       "standard": 46.26865671641791
      }
     },
     "sealed|score": {
      "cc": 30.594133728758123,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 51.52640350877192,
       "hard": 23.075859154929613,
       "judge": 30.547135416666638,
       "standard": 35.23504545454546
      }
     }
    }
   },
   "jevbench_score": 60.010819075526705,
   "key": "amazon-strands-decider-26b-a4b",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 checkpoint (model card and LICENSE.md); public, not gated; base google/gemma-4-26B-A4B-it: Apache-2.0 per preregistered model metadata",
   "listing": "ranked",
   "measured_in": "v1.7.1",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "amazon/strands-decider-26B-A4B-gemma4-v1-2610@f08dc85955d008dc247fae5b4df53bc409704f8a",
   "new_in": "v1.7.1",
   "noul_decisive": {
    "acc_among_decisive": 0.9144981412639405,
    "decisive": 269,
    "decisive_rate": 0.7211796246648794,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 110,
   "ranked": true,
   "ranks": {
    "A": 110,
    "B": 111,
    "C": 114,
    "capability": 121
   },
   "repo": "https://huggingface.co/amazon/strands-decider-26B-A4B-gemma4-v1-2610",
   "scores": {
    "A": 60.010819075526705,
    "B": 54.848762321396165,
    "C": 41.67417991356022
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3711168100000123,
    "p50_s_raw": 0.11055840500000613,
    "p95_s_adjusted": 0.43073743310000057,
    "p95_s_raw": 0.14036871655000027
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "First measurement of Amazon / AWS Strands Decider, a LoRA r48/alpha96 plus pointer readout head on google/gemma-4-26B-A4B-it at 4d7ae4984b7db7de8f8457170b3f1a419ee76d52. Author's own unmodified strands-labs/strands-decider server at b81d443c07438dc5ed51d0e85f9db184ad65d809, peft 0.21.0, BF16 CUDA default attention; vLLM was not imported. Native choice, noul and score outputs, text only, temperature 0.702122181653982. Strict 4096-token window, no truncation: 1485 answers and 15 native HTTP 422 refusals at the predeclared long positions. Refusals count wrong in the full 1500-item capability denominator. First 50 responses repeated bit-identically. The author reports using the public JevBench set for development; training disclosure lists public Hub datasets, ContractNLI/MuSiQue, synthetic/code rows and Qwen teacher distributions. No overlap or contamination audit was run. Public-set evaluation used an older code revision; parity with it is unproven. Earlier create/cache-verifier failures stopped before held-out input and were never scored; only the successful second VM measurement is used. The cache preparation fix did not change weights, serving code, prompts, refusal set or scoring.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 61.49222222222223,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": -7.189999999999985
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 61.49222222222223
      },
      "calibration_gen": {
       "accuracy": 0.5643564356435643,
       "cc": 29.13501458112408,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": 4.0000000000000036,
        "score": 27.94375
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 29.13501458112408
      },
      "composite_gen": {
       "accuracy": 0.6435643564356436,
       "cc": 31.348934403446595,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -11.111111111111116,
        "score": 21.418076923076924
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 31.348934403446595
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 75.25648148148149,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 65.76944444444445
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 75.25648148148149
      },
      "jaggedness": {
       "accuracy": 0.5611111111111111,
       "cc": 32.00741758063334,
       "cc_by_type": {
        "choice": 50.9143407122233,
        "noul": 8.108108108108114,
        "score": 36.999803921568606
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 32.00741758063334
      },
      "llm_router": {
       "accuracy": 0.7539936102236422,
       "cc": 62.25702374029731,
       "cc_by_type": {
        "choice": 71.47133926212908,
        "noul": 75.25773195876289,
        "score": 40.04199999999999
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 62.25702374029731
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": 7.909855997381771,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": 0,
        "score": -14.414761904761896
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 7.909855997381771
      },
      "multilingual": {
       "accuracy": 0.730909090909091,
       "cc": 53.5875480313683,
       "cc_by_type": {
        "choice": 73.58982719269645,
        "noul": 41.66666666666667,
        "score": 45.50615023474178
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 53.5875480313683
      },
      "negation_pair_gen": {
       "accuracy": 0.5697674418604651,
       "cc": 8.676555555555545,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": -66.66666666666667,
        "score": 9.362999999999976
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 8.676555555555545
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 77.28637560637561,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100,
        "score": 58.88615384615385
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 77.28637560637561
      },
      "usecase": {
       "accuracy": 0.6409090909090909,
       "cc": 41.547845228758185,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 25,
        "score": 26.604320000000026
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 41.547845228758185
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 14.97722832722831,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": -66.97307692307697
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 14.97722832722831
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 51.47211111111111,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 21.083000000000006
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 51.47211111111111
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 69.11111068359625,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75,
        "score": 64.19173913043477
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 69.11111068359625
      },
      "el": {
       "accuracy": 0.6,
       "cc": 40.45903786468032,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -33.333333333333336,
        "score": 88.22999999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 40.45903786468032
      },
      "en": {
       "accuracy": 0.6724587315377932,
       "cc": 42.778160345311846,
       "cc_by_type": {
        "choice": 71.00352692791152,
        "noul": 25.179856115107913,
        "score": 32.15109799291609
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 42.778160345311846
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 67.74794549467423,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 13.031764705882365
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.74794549467423
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 38.31978142076502,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 79.98666666666665
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 38.31978142076502
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 47.57486068111456,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 63.901052631578956
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 47.57486068111456
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 32.33598230912476,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 50,
        "score": -19.472500000000025
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 32.33598230912476
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 59.23025461158742,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 5.7499999999999885
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 59.23025461158742
      },
      "ja": {
       "accuracy": 0.76,
       "cc": 56.431292560233,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 80,
        "score": 17.888000000000005
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 56.431292560233
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 72.33232804232803,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100,
        "score": 61.44142857142856
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 72.33232804232803
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 70.90555555555555,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 79.38333333333333
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 70.90555555555555
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 43.0344357864358,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 33.33333333333333,
        "score": 21.744000000000018
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 43.0344357864358
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 49.40628333448152,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 43.453720930232556
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 49.40628333448152
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 49.4102388782228,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 60.00000000000001,
        "score": 41.26548387096774
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 49.4102388782228
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 17.702056737588652,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -19.999999999999996,
        "score": 24.17
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 17.702056737588652
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 58.90028985507246,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": -1.560000000000028
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 58.90028985507246
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 85.21731182795698,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 87.91
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 85.21731182795698
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 49.31653483992468,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": 40.040000000000006
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 49.31653483992468
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.776,
       "cc": 71.06718459978187,
       "cc_by_type": {
        "choice": 71.06718459978187
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 71.06718459978187
      },
      "noul": {
       "accuracy": 0.656,
       "cc": 31.200000000000006,
       "cc_by_type": {
        "noul": 31.200000000000006
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 31.200000000000006
      },
      "score": {
       "accuracy": 0.5226666666666666,
       "cc": 33.56303249097474,
       "cc_by_type": {
        "score": 33.56303249097474
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 33.56303249097474
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -13.746666666666664,
       "cc_by_type": {
        "choice": 36,
        "noul": -75,
        "score": -2.2399999999999975
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.7083333333333334,
       "cc": 49.0915878324574,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 100,
        "score": 3.1753846153846133
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 49.0915878324574
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 34.966558265582655,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": 37.42000000000001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 34.966558265582655
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -15.041444931773883,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": -18.223750000000006
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 22.636011473429956,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 30.226874999999996
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.636011473429956
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 58.86036687631027,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": 68.405
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 58.86036687631027
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 38.40946132299073,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 20.55051282051281
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 38.40946132299073
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": -7.8120375700651365,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100,
        "score": 28.82869565217392
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 40.2212962962963,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 19.999999999999996,
        "score": 21.497222222222234
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 40.2212962962963
      },
      "knowledge_graphs": {
       "accuracy": 0.45454545454545453,
       "cc": 26.202986394999243,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 33.33333333333333,
        "score": 9.51545454545455
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 26.202986394999243
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 54.176017539738474,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": -3.6218181818181483
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 54.176017539738474
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 3.382629928315404,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -26.41125000000002
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 3.382629928315404
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 51.34694444444444,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 34.04083333333332
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 51.34694444444444
      },
      "model_routing": {
       "accuracy": 0.7695852534562212,
       "cc": 66.03566829004329,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 90.90909090909092,
        "score": 28.236874999999984
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.03566829004329
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 83.62882352941178,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 50.88647058823531
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 83.62882352941178
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 58.402136752136755,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 41.87307692307694
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 58.402136752136755
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 28.33601854534918,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": 14.564705882352946
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 28.33601854534918
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 44.67394202898549,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 29.673999999999978
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 44.67394202898549
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 76.49640350877193,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 79.48921052631577
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 76.49640350877193
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.18693132697233,
      87.71265054206363
     ],
     "I": [
      41.25486066062294,
      51.100250740648384
     ],
     "capability": [
      62.15530023104903,
      68.3584931229888
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.70498962717834,
     "capability": 66.71562929560538,
     "intelligence": 46.72626896403243
    },
    "per_type": {
     "choice": {
      "accuracy": 0.776,
      "calibration": 89.1862950819672,
      "cc": 71.06718459978187,
      "cc_by_type": {
       "choice": 71.06718459978187
      },
      "cc_tiered": 70.53092369038364,
      "cc_tiered_open": 70.08897277377838,
      "cc_tiered_sealed": 70.97287460698891,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 71.06718459978187
     },
     "noul": {
      "accuracy": 0.656,
      "calibration": 82.81276139410187,
      "cc": 31.200000000000006,
      "cc_by_type": {
       "noul": 31.200000000000006
      },
      "cc_tiered": 33.95863451915277,
      "cc_tiered_open": 46.2415458937198,
      "cc_tiered_sealed": 21.675723144585735,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 31.200000000000006
     },
     "score": {
      "accuracy": 0.5226666666666666,
      "calibration": 88.11591240546593,
      "cc": 33.56303249097474,
      "cc_by_type": {
       "score": 33.56303249097474
      },
      "cc_tiered": 35.68924868256088,
      "cc_tiered_open": 40.784363636363636,
      "cc_tiered_sealed": 30.594133728758123,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 33.56303249097474
     }
    },
    "raw_on_A": {
     "calibration": 84.48357098456802,
     "capability": 66.33965771143644,
     "intelligence": 48.19574443830487
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 56.128718989169144,
    "Emphasis on Accuracy 60:20:20": 48.80010375835342,
    "Emphasis on Cost 20:20:60": 58.61594759102552,
    "Emphasis on Speed 20:60:20": 62.90649048361825,
    "Intelligence only": 40.80781154329
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Scira (zaid@scira.ai)",
   "axes": {
    "calibration": 87.80739845794042,
    "cost": 30.04968825138303,
    "intelligence": 68.17020573384414,
    "speed": 83.27058287283708
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.056815051399762694,
      "ece_score": 88.63698972004747,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.21879788452671,
      "score_before_time_limit": 88.63698972004747,
      "time_limit_failed_share": 0.016
     },
     "noul": {
      "abstention_rate": 0.15200000000000002,
      "brier": 0.06134542488590291,
      "ece": 0.061674600501855234,
      "n": 375,
      "score": 86.7299857140329,
      "score_before_time_limit": 87.66507989962895,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.9866666666666667,
      "interval90_width": 0.30288888888888893,
      "n": 375,
      "nrps": 0.054088924234169763,
      "score": 89.47341177526164,
      "score_before_time_limit": 90.6825119343868,
      "time_limit_failed_share": 0.013333333333333334,
      "top_ece": 0.06613041853904722
     }
    },
    "score": 87.80739845794042
   },
   "capability": 77.98880209589228,
   "composite_ci95": {
    "A": [
     19.823210414087278,
     20.435488221185672
    ],
    "B": [
     20.365620398737217,
     21.348362906693566
    ],
    "C": [
     19.823210414087278,
     20.435488221185672
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), exact base-model market reference from the frozen 25 Sep 2026 snapshot (pricing_v15 sha256 dae86740...): qwen/qwen3.8-27b USD 0.42/M input, USD 3.0/M output, x the provider-reported input tokens; free keyless endpoint, nothing generated (output 0). OVER THE OFFICIAL COST CAP, DISCLOSED: USD 0.214623 per 1,000 decisions on the common basis (USD 0.542843 over all 1,500 items) is 3.323x the official cap USD 0.064595; the row is ranked (v1.6.16 vllm-sr-d3 precedent) and falls outside the official-caps board views by construction. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.214623 per 1,000 decisions (3.323x the official cost cap USD 0.064595), against USD 0.542843 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.5428432799999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.21462339393939392
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 21,
    "n_over_limit_answered_ok": 21,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Amos 1 large (Scira)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.amos.scira.ai /v1/systemone API, model `amos-1-large`; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `scira` before the first request (every sealed item of this draw was already retired). Native probabilities on all three question types, no generated tokens. Hosting implementation and deployed weights were not independently inspected. Free, keyless; 1 s spacing for the documented 60 requests/minute.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 70.71062454329342,
    "I_sealed": 65.62978692439485,
    "base": 68.17020573384414,
    "excess": -4.7586891514571406,
    "gap": 5.08083761889857,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 83.93848091567699,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 83.01352626612143,
       "judge": 84.17930125247199,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 69.17046238785369,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 39.99999999999999,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 59.02293032634961,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 69.3524417552081,
       "hard": 68.76756728678528,
       "judge": 55.49798398673283,
       "standard": 39.65632020047416
      }
     },
     "sealed|choice": {
      "cc": 83.05942075029978,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 77.62229675031803,
       "judge": 84.1332027424094,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 57.620837217337474,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 47.272727272727266,
       "judge": 53.846153846153854,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 56.20910280554729,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 79.12768431971863,
       "hard": 54.857051756337924,
       "judge": 53.445570314943325,
       "standard": 51.5992128827863
      }
     }
    }
   },
   "jevbench_score": 20.25053358459254,
   "key": "amos-1-large",
   "last_measured_on": "2026-10-13",
   "licence": "Apache-2.0 open weights declared by the submitter (mukaddamzaid/amos-1-large); the measured object is Scira's hosted endpoint, whose deployed weights were not inspected",
   "listing": "ranked",
   "measured_in": "v1.6.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.amos.scira.ai/v1/systemone#amos-1-large",
   "noul_decisive": {
    "acc_among_decisive": 0.9651898734177216,
    "decisive": 316,
    "decisive_rate": 0.8517520215633423,
    "n": 375,
    "supported": true,
    "valid": 371
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 35,
    "axes": {
     "calibration": 88.99486051802106,
     "cost": 30.04968825138303,
     "intelligence": 69.72341288828827,
     "speed": 83.61644312651299
    },
    "board_release": "v1.6.43",
    "capability": 79.35913670315466,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.000010539179282,
      20.572867919013834
     ],
     "B": [
      20.620413647103454,
      21.54072052065241
     ],
     "C": [
      20.000010539179282,
      20.572867919013834
     ]
    },
    "jevbench_score": 20.401636755316044,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 130,
    "ranks": {
     "A": 130,
     "B": 123,
     "C": 112,
     "capability": 27
    },
    "scores": {
     "A": 20.401636755316044,
     "B": 21.207017501806387,
     "C": 20.401636755316044
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.5090118609950878,
     "p50_s_raw": 0.5090118609950878,
     "p95_s_adjusted": 0.8543341861019143,
     "p95_s_raw": 0.8543341861019143
    },
    "views": {
     "Balanced 33:33:33": 18.186993256482154,
     "Emphasis on Accuracy 60:20:20": 20.460803958388272,
     "Emphasis on Cost 20:20:60": 14.317576145069761,
     "Emphasis on Speed 20:60:20": 21.628687597892217,
     "Intelligence only": 25.18364392366508
    }
   },
   "rank": 186,
   "ranked": true,
   "ranks": {
    "A": 186,
    "B": 181,
    "C": 170,
    "capability": 45
   },
   "repo": "https://huggingface.co/mukaddamzaid/amos-1-large",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 20.25053358459254,
    "B": 20.996167711057787,
    "C": 20.25053358459254
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.532363753503887,
    "p50_s_raw": 0.532363753503887,
    "p95_s_adjusted": 0.8845720937955772,
    "p95_s_raw": 0.8845720937955772
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 85.70884085363811,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 73.79318922758102
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 85.70884085363811
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 50.47615368112714,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 43.99999999999999,
        "score": 32.879150332883
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 50.47615368112714
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 86.67411659317217,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 70.68178970724917
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 86.67411659317217
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.2317845951076,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 89.69535378532278
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.2317845951076
      },
      "jaggedness": {
       "accuracy": 0.7222222222222222,
       "cc": 52.46040101820392,
       "cc_by_type": {
        "choice": 66.79499518768047,
        "noul": 29.729729729729737,
        "score": 60.85647813720154
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 52.46040101820392
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 83.32430365063594,
       "cc_by_type": {
        "choice": 95.24522321035484,
        "noul": 95.8762886597938,
        "score": 58.85139908175916
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.32430365063594
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7037037037037037,
       "cc": 43.067969834272596,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 25,
        "score": 53.68844558529201
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 43.067969834272596
      },
      "multilingual": {
       "accuracy": 0.7818181818181819,
       "cc": 61.422268079528465,
       "cc_by_type": {
        "choice": 81.41506358004565,
        "noul": 44.44444444444444,
        "score": 58.407296214095304
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 61.422268079528465
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 75.73214123542938,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 60.529757039621465
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 75.73214123542938
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 81.14468880284434,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 55.44607842054504
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 81.14468880284434
      },
      "usecase": {
       "accuracy": 0.7772727272727272,
       "cc": 60.36687776140042,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 41.66666666666667,
        "score": 60.267299950867894
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 60.36687776140042
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 61.882864079814546,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100,
        "score": 23.743830334681725
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 61.882864079814546
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 52.73782751626438,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 24.880149215459824
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 52.73782751626438
      },
      "de": {
       "accuracy": 0.875,
       "cc": 83.90096934595631,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 75.59671334760345
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.90096934595631
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 53.05558601212194,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 92.6863111089915
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 53.05558601212194
      },
      "en": {
       "accuracy": 0.8340573414422241,
       "cc": 69.82585015619428,
       "cc_by_type": {
        "choice": 84.62308246177126,
        "noul": 66.18705035971225,
        "score": 58.6674176470993
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 69.82585015619428
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 70.72155656154153,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 21.952597906484318
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 70.72155656154153
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 74.873681863149,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 91.28771225611368
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 74.873681863149
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 81.05883440654436,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 85.52944439610368
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 81.05883440654436
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 61.66610015218894,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 50,
        "score": 68.51785352919251
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 61.66610015218894
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 67.97700324005393,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 17.96062780278067
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 67.97700324005393
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 43.73893493579494,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 11.216804807384817
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 43.73893493579494
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 11.692657175340818,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 35.077971526022466
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 11.692657175340818
      },
      "mixed": {
       "accuracy": 1,
       "cc": 97.24507216985027,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 91.73521650955081
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 97.24507216985027
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -17.41688895535159,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": -26.276640892028812
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 75,
       "cc_by_type": {
        "choice": 100,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 75
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 77.465615091364,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80,
        "score": 60.01428073748598
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 77.465615091364
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 63.573232278872894,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 62.54055717839828
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 63.573232278872894
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 56.68991453443647,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 61.13357339054346
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 56.68991453443647
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 67.58806742727757,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 52.764202281832695
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 67.58806742727757
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 80.91469064595238,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 75.00213645398617
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 80.91469064595238
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 70.9322772713171,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 79.463498480618
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 70.9322772713171
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8746666666666667,
       "cc": 83.8114009070208,
       "cc_by_type": {
        "choice": 83.8114009070208
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 83.8114009070208
      },
      "noul": {
       "accuracy": 0.8133333333333334,
       "cc": 62.66666666666667,
       "cc_by_type": {
        "noul": 62.66666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 62.66666666666667
      },
      "score": {
       "accuracy": 0.7333333333333333,
       "cc": 57.91409320050066,
       "cc_by_type": {
        "score": 57.91409320050066
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 57.91409320050066
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 12.801234460928859,
       "cc_by_type": {
        "choice": 52,
        "noul": -50,
        "score": 36.40370338278658
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 12.801234460928859
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 83.81931093174488,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 70.09147316790545
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 83.81931093174488
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 25.116387537117735,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": 7.869487814605236
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 25.116387537117735
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -4.393060324753523,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": -52.94526284560559
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 40.641138161384106,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 69.74950144067407
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 40.641138161384106
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": -4.760219093872943,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": -122.45675791054964
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.8,
       "cc": 54.586939841529194,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 14.28571428571428,
        "score": 59.27902680750074
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 54.586939841529194
      },
      "gaming": {
       "accuracy": 0.9090909090909091,
       "cc": 71.2806493361724,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50,
        "score": 74.29490968099105
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 71.2806493361724
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 16.02023316616262,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -19.999999999999996,
        "score": 30.560699498487853
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 16.02023316616262
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 67.5820202385729,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 61.77532552656804
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 67.5820202385729
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 62.07152141651769,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 42.2869156707417
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 62.07152141651769
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": 0.9993600672424611,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": -1.3029950670898183
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0.9993600672424611
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 73.86169788565626,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 61.58509365696874
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 73.86169788565626
      },
      "model_routing": {
       "accuracy": 0.9170506912442397,
       "cc": 81.67870057339692,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 100,
        "score": 49.711426395515446
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.67870057339692
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 19.51262058240074,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 70.05417652647284
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 19.51262058240074
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 57.303088588281206,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 72.00211627552143
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.303088588281206
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 26.574834589572514,
       "cc_by_type": {
        "choice": 100,
        "noul": -50,
        "score": 29.724503768717547
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 26.574834589572514
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 48.40292921906184,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": 0,
        "score": 59.701541280373924
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 48.40292921906184
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 97.1544132796688,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 91.46323983900642
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.1544132796688
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.54175553237744,
      88.18443530250994
     ],
     "I": [
      63.229435649690274,
      72.06788091953597
     ],
     "capability": [
      74.10480215942516,
      79.41671065887567
     ],
     "composite": {
      "A": [
       19.823210414087278,
       20.435488221185672
      ],
      "B": [
       20.365620398737217,
       21.348362906693566
      ],
      "C": [
       19.823210414087278,
       20.435488221185672
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.80739845794042,
     "capability": 77.98880209589228,
     "intelligence": 68.17020573384414
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8746666666666667,
      "calibration": 87.21879788452671,
      "cc": 83.8114009070208,
      "cc_by_type": {
       "choice": 83.8114009070208
      },
      "cc_tiered": 83.49895083298838,
      "cc_tiered_open": 83.93848091567699,
      "cc_tiered_sealed": 83.05942075029978,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.8114009070208
     },
     "noul": {
      "accuracy": 0.8133333333333334,
      "calibration": 86.7299857140329,
      "cc": 62.66666666666667,
      "cc_by_type": {
       "noul": 62.66666666666667
      },
      "cc_tiered": 63.395649802595585,
      "cc_tiered_open": 69.17046238785369,
      "cc_tiered_sealed": 57.620837217337474,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 62.66666666666667
     },
     "score": {
      "accuracy": 0.7333333333333333,
      "calibration": 89.47341177526164,
      "cc": 57.91409320050066,
      "cc_by_type": {
       "score": 57.91409320050066
      },
      "cc_tiered": 57.61601656594845,
      "cc_tiered_open": 59.02293032634961,
      "cc_tiered_sealed": 56.20910280554729,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 57.91409320050066
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.016000000000000014,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.072801136912016,
    "Emphasis on Accuracy 60:20:20": 20.224788770596167,
    "Emphasis on Cost 20:20:60": 14.2749725946048,
    "Emphasis on Speed 20:60:20": 21.50613973787975,
    "Intelligence only": 24.622635586625112
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Scira (zaid@scira.ai)",
   "axes": {
    "calibration": 87.60140992244203,
    "cost": 48.74716696332005,
    "intelligence": 38.09902514689999,
    "speed": 86.2337589756876
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06666585481947378,
      "ece_score": 86.66682903610524,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 85.51127131562384,
      "score_before_time_limit": 86.66682903610524,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.43999999999999995,
      "brier": 0.12374828347808393,
      "ece": 0.06865945905447007,
      "n": 375,
      "score": 86.26810818910599,
      "score_before_time_limit": 86.26810818910599,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9413333333333334,
      "interval90_width": 0.4971111111111112,
      "n": 375,
      "nrps": 0.09858810096196241,
      "score": 91.02485026259626,
      "score_before_time_limit": 91.02485026259626,
      "time_limit_failed_share": 0,
      "top_ece": 0.04045744689305625
     }
    },
    "score": 87.60140992244203
   },
   "capability": 62.85021753467101,
   "composite_ci95": {
    "A": [
     21.482358613369826,
     41.7184973132223
    ],
    "B": [
     18.994596231491883,
     38.79119242357073
    ],
    "C": [
     14.918304592617936,
     28.971178689737705
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), exact base-model market reference from the frozen 25 Sep 2026 snapshot (pricing_v15 sha256 dae86740...): qwen/qwen3.5-9b USD 0.09999999999999999/M input, USD 0.15/M output, x the provider-reported input tokens; free keyless endpoint, nothing generated (output 0). INSIDE THE OFFICIAL COST CAP on the common basis: USD 0.051101 per 1,000 decisions is 0.791x the official cap USD 0.064595. Over all 1,500 items the row cost USD 0.129248 (2.001x the cap) and was disclosed as ranked over the cap in v1.6.29; that disclosure is superseded. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.051101 per 1,000 decisions (0.791x the official cost cap USD 0.064595), against USD 0.129248 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.12924839999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05110080808080808
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 10,
    "n_over_limit_answered_ok": 10,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Amos 1 medium (Scira)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.amos.scira.ai /v1/systemone API, model `amos-1-medium`; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `scira` before the first request (every sealed item of this draw was already retired). Native probabilities on all three question types, no generated tokens. Hosting implementation and deployed weights were not independently inspected. Free, keyless; 1 s spacing for the documented 60 requests/minute.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 43.6624142098521,
    "I_sealed": 32.53563608394788,
    "base": 38.09902514689999,
    "excess": 1.2872513555485128,
    "gap": 11.126778125904224,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 69.12490790461862,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 64.1396665618119,
       "judge": 80.22412656558998,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 22.332643202208423,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 30.434782608695656,
       "judge": 0,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 39.52969152272925,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.14080159637061,
       "hard": 44.43989097468418,
       "judge": 36.67618524977716,
       "standard": 30.18399699142684
      }
     },
     "sealed|choice": {
      "cc": 62.60628592189268,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 60.969122238926786,
       "judge": 60.33300685602351,
       "standard": 62.982493144906144
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 31.034482758620683,
       "hard": -9.090909090909093,
       "judge": -16.92307692307692,
       "standard": 25.373134328358216
      }
     },
     "sealed|score": {
      "cc": 35.00062232995097,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 49.394405829279044,
       "hard": 31.30311505659157,
       "judge": 33.35206227687498,
       "standard": 37.67158520661971
      }
     }
    }
   },
   "jevbench_score": 31.638672185087426,
   "key": "amos-1-medium",
   "last_measured_on": "2026-10-13",
   "licence": "Apache-2.0 open weights declared by the submitter (mukaddamzaid/amos-1-medium); the measured object is Scira's hosted endpoint, whose deployed weights were not inspected",
   "listing": "ranked",
   "measured_in": "v1.6.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.amos.scira.ai/v1/systemone#amos-1-medium",
   "noul_decisive": {
    "acc_among_decisive": 0.9714285714285714,
    "decisive": 210,
    "decisive_rate": 0.56,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 31,
    "axes": {
     "calibration": 87.98659582926916,
     "cost": 48.74716696332005,
     "intelligence": 38.36362733252365,
     "speed": 86.48270625779318
    },
    "board_release": "v1.6.43",
    "capability": 63.175111580896406,
    "capability_api_board_rank": 24,
    "capability_open_board_rank": null,
    "capability_rank": 69,
    "composite_ci95": {
     "A": [
      22.106438163001933,
      42.39900600898038
     ],
     "B": [
      19.591717479749285,
      39.302428856817606
     ],
     "C": [
      15.351693168751336,
      29.44375417290304
     ]
    },
    "jevbench_score": 32.20168753691601,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 96,
    "ranks": {
     "A": 96,
     "B": 98,
     "C": 98,
     "capability": 111
    },
    "scores": {
     "A": 32.20168753691601,
     "B": 29.274084226551246,
     "C": 22.362283011747234
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.44348932350840187,
     "p50_s_raw": 0.44348932350840187,
     "p95_s_adjusted": 0.5068112015054794,
     "p95_s_raw": 0.5068112015054794
    },
    "views": {
     "Balanced 33:33:33": 28.872147628956235,
     "Emphasis on Accuracy 60:20:20": 25.371533776069526,
     "Emphasis on Cost 20:20:60": 28.21250381149062,
     "Emphasis on Speed 20:60:20": 34.42715214171818,
     "Intelligence only": 21.46731530678655
    }
   },
   "rank": 159,
   "ranked": true,
   "ranks": {
    "A": 159,
    "B": 160,
    "C": 161,
    "capability": 145
   },
   "repo": "https://huggingface.co/mukaddamzaid/amos-1-medium",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 31.638672185087426,
    "B": 28.73772686231222,
    "C": 21.97130012853293
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.44756228550249944,
    "p50_s_raw": 0.44756228550249944,
    "p95_s_adjusted": 0.5318272908531071,
    "p95_s_raw": 0.5318272908531071
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 70.72841647184556,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 28.851916082203388
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 70.72841647184556
      },
      "calibration_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 14.975090543066846,
       "cc_by_type": {
        "choice": 36.37327677624603,
        "noul": 4.0000000000000036,
        "score": 4.551994852954511
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 14.975090543066846
      },
      "composite_gen": {
       "accuracy": 0.49504950495049505,
       "cc": 6.091925072879623,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -62.96296296296296,
        "score": 13.759063384853864
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 6.091925072879623
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 91.49611264605214,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 74.48833793815639
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 91.49611264605214
      },
      "jaggedness": {
       "accuracy": 0.5111111111111111,
       "cc": 12.733145931714104,
       "cc_by_type": {
        "choice": 45.13955726660251,
        "noul": -40.54054054054054,
        "score": 33.60042106908033
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 12.733145931714104
      },
      "llm_router": {
       "accuracy": 0.7827476038338658,
       "cc": 63.07665363153567,
       "cc_by_type": {
        "choice": 71.47133926212908,
        "noul": 62.88659793814433,
        "score": 54.87202369433361
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 63.07665363153567
      },
      "long_state": {
       "accuracy": 0.2,
       "cc": -3.9955246973262035,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 0,
        "score": 17.045683972537518
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -13.788806870177382,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -75,
        "score": -16.881884528057924
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6618181818181819,
       "cc": 40.822452938700465,
       "cc_by_type": {
        "choice": 70.6553635474405,
        "noul": 5.555555555555558,
        "score": 46.256439713105344
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 40.822452938700465
      },
      "negation_pair_gen": {
       "accuracy": 0.5,
       "cc": -6.27999295377069,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -80,
        "score": -12.173312194645414
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 72.02153229907263,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 100,
        "score": 49.09762993025094
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 72.02153229907263
      },
      "usecase": {
       "accuracy": 0.5954545454545455,
       "cc": 33.58098123168537,
       "cc_by_type": {
        "choice": 63.23529411764707,
        "noul": 0,
        "score": 37.50764957740904
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 33.58098123168537
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -5.625522909037805,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0,
        "score": -59.73371158425627
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 49.37440980616833,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 35.52722327411174
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 49.37440980616833
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 86.26833998280311,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 74.73422348823236
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 86.26833998280311
      },
      "el": {
       "accuracy": 0.4,
       "cc": 14.87006399662579,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100,
        "score": 94.88952159881592
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 14.87006399662579
      },
      "en": {
       "accuracy": 0.629887054735013,
       "cc": 34.99976613724583,
       "cc_by_type": {
        "choice": 62.65605740715879,
        "noul": 6.474820143884896,
        "score": 35.8684208606938
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 34.99976613724583
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 62.123511307964144,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -13.629466076107576
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 62.123511307964144
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 36.950170300317374,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 76.97072948018709
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 36.950170300317374
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 46.162580842492474,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 80.84068370394802
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 46.162580842492474
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 15.044394910666796,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -25,
        "score": -13.107038731686771
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 15.044394910666796
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 59.21561631194971,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -8.32353298153199
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 59.21561631194971
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 40.035094113356514,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 19.999999999999996,
        "score": 28.699404659370586
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 40.035094113356514
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -0.99009225371654,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 41.47416768329484
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.0593334229456,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 91.84466693550348
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.0593334229456
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -6.718549297485519,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 31.79240405559539
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 45.495999792566494,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 31.72287030448745
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 45.495999792566494
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.80789114038846,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": 65.45844065746472
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.80789114038846
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 17.008028029074485,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 47.61982876807451
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 17.008028029074485
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 63.07036542482134,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 10.950226709246625
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 63.07036542482134
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.65635733546749,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 59.22713652253151
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.65635733546749
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 1.6406301152936227,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -36.32104750722647
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 1.6406301152936227
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7253333333333334,
       "cc": 64.52285730687538,
       "cc_by_type": {
        "choice": 64.52285730687538
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 64.52285730687538
      },
      "noul": {
       "accuracy": 0.544,
       "cc": 8.800000000000008,
       "cc_by_type": {
        "noul": 8.800000000000008
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 8.800000000000008
      },
      "score": {
       "accuracy": 0.5653333333333334,
       "cc": 36.795063482577376,
       "cc_by_type": {
        "score": 36.795063482577376
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.795063482577376
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -11.612698505906499,
       "cc_by_type": {
        "choice": 20,
        "noul": -50,
        "score": -4.838095517719498
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 24.522153858600834,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 11.111111111111116,
        "score": -18.911109162637807
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 24.522153858600834
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 30.995651582372062,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50,
        "score": 8.027605153620243
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 30.995651582372062
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -21.304913464984477,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -100,
        "score": -5.435208231210731
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 19.91667593401915,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 36.56162200495601
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 19.91667593401915
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -23.035431154993848,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -33.333333333333336,
        "score": -85.45849472284317
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 21.961594726546508,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -42.85714285714286,
        "score": 38.15369174266473
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 21.961594726546508
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": -2.6028106451585344,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100,
        "score": 44.456376426893726
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 14.729500546085612,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60,
        "score": 45.85516830492351
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 14.729500546085612
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 18.37776298144904,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 0,
        "score": 32.2210833768953
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 18.37776298144904
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 37.43981895623384,
       "cc_by_type": {
        "choice": 100,
        "noul": -11.111111111111116,
        "score": 23.430567979812633
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 37.43981895623384
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -26.159304338357146,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -13.96178398281338
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 26.337856947168092,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 33.56251517931618
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 26.337856947168092
      },
      "model_routing": {
       "accuracy": 0.8341013824884793,
       "cc": 69.75791357046573,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 87.87878787878789,
        "score": 41.265082702739164
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 69.75791357046573
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 12.28249541563279,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 48.363801026168986
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 12.28249541563279
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 62.49576394692955,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 54.15395850745532
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 62.49576394692955
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 41.45276807709171,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": 39.13662935442785
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 41.45276807709171
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 10.829423826169853,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 38.28537292778492
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 10.829423826169853
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 77.41282379352732,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 82.23847138058198
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 77.41282379352732
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.79575916328675,
      88.21506895592145
     ],
     "I": [
      32.48961964612892,
      43.00911209428756
     ],
     "capability": [
      58.2975259643926,
      64.52717043871775
     ],
     "composite": {
      "A": [
       21.482358613369826,
       41.7184973132223
      ],
      "B": [
       18.994596231491883,
       38.79119242357073
      ],
      "C": [
       14.918304592617936,
       28.971178689737705
      ]
     },
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.60140992244203,
     "capability": 62.85021753467101,
     "intelligence": 38.09902514689999
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7253333333333334,
      "calibration": 85.51127131562384,
      "cc": 64.52285730687538,
      "cc_by_type": {
       "choice": 64.52285730687538
      },
      "cc_tiered": 65.86559691325564,
      "cc_tiered_open": 69.12490790461862,
      "cc_tiered_sealed": 62.60628592189268,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 64.52285730687538
     },
     "noul": {
      "accuracy": 0.544,
      "calibration": 86.26810818910599,
      "cc": 8.800000000000008,
      "cc_by_type": {
       "noul": 8.800000000000008
      },
      "cc_tiered": 11.166321601104212,
      "cc_tiered_open": 22.332643202208423,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 8.800000000000008
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 91.02485026259626,
      "cc": 36.795063482577376,
      "cc_by_type": {
       "score": 36.795063482577376
      },
      "cc_tiered": 37.26515692634011,
      "cc_tiered_open": 39.52969152272925,
      "cc_tiered_sealed": 35.00062232995097,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.795063482577376
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 28.370624106486808,
    "Emphasis on Accuracy 60:20:20": 24.892625409924076,
    "Emphasis on Cost 20:20:60": 27.764641557653366,
    "Emphasis on Speed 20:60:20": 33.836817745655395,
    "Intelligence only": 21.02617789811459
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "EmpirioLabs (@Adam-Dalloul)",
   "axes": {
    "calibration": 88.94185474430773,
    "cost": 72.4070134176205,
    "intelligence": 45.23340779864823,
    "speed": 88.75158098066062
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06972206212499996,
      "ece_score": 86.055587575,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 85.7113652247,
      "score_before_time_limit": 86.055587575,
      "time_limit_failed_share": 0.004
     },
     "noul": {
      "abstention_rate": 0.31733333333333336,
      "brier": 0.11392156247660881,
      "ece": 0.04671065970666656,
      "n": 375,
      "score": 90.41611374384358,
      "score_before_time_limit": 90.6578680586667,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.9706666666666667,
      "interval90_width": 0.4937777777777778,
      "n": 375,
      "nrps": 0.09021695431167416,
      "score": 90.69808526437963,
      "score_before_time_limit": 90.94059351374963,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.045485587706666636
     }
    },
    "score": 88.94185474430773
   },
   "capability": 67.08763127147799,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,249,558 input + 0 output tokens over 1,477 decisions = 846.010832769 tokens/decision. GPU-time cost USD 0.00831316153405032/1,000; old list-price cost USD 0.0253803249830738/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00831316153405032/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.08184117999999999
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.008313161534050324
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 5,
    "n_over_limit_answered_ok": 5,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Aplomb 1 (5.3B decision model on Qwen3.5-4B, trained readout head, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights empiriolabsai/aplomb-1@f8e2dc81f0505db21a3b7a933ea479658e4ebb8f, bf16, 5.3 B, base Qwen/Qwen3.5-4B plus the shipped 1.3 GB audio encoder and the trained DecisionHead; click-through gate accepted under the EmpirioLabs Model License 1.0 section 2.1, which grants evaluation and the publication of benchmark results without asking. Served behind the author's own TypeSafe wire format by a TRANSPORT SHIM ONLY: run_aplomb.main()'s loading block verbatim, every request handed to THEIR run_aplomb.decide() with THEIR edm_config.json temperatures and abstain_bias and debias=True - the configuration the author names as the reference in #195 and in the model card. Their own aplomb/schema.py documents the request schema as wire-compatible with TypeSafe /v1/systemone and their aplomb/readout.py already emits the answer dicts the unchanged typesafe adapter parses, so no answer field is rewritten and no contributor file is edited. STACK: the author's own documented install line, with transformers pinned to 5.17.0 - the version their own audio sidecar was saved with; their loader compares the saved config to the live one and 5.19.0 fails it on the transformers_version string alone. SPEED CAVEAT: their documented install line does not include causal_conv1d, so transformers falls back to a reference PyTorch implementation it itself calls much slower; the Speed axis is measured on the stack the author documents. One H100 80 GB (Lium). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 55, GitHub #195",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 48.14750387886506,
    "I_sealed": 42.31931171843139,
    "base": 45.23340779864823,
    "excess": 3.254804517995422,
    "gap": 5.828192160433666,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.2456274917742,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 67.9144385026738,
       "judge": 80.22412656558997,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 26.777087646652866,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 30.434782608695656,
       "judge": 0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 47.419796498168125,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 57.31956009090909,
       "hard": 46.33070064814815,
       "judge": 45.652502593220326,
       "standard": 47.299047259259254
      }
     },
     "sealed|choice": {
      "cc": 61.23201765527931,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 56.730769230769226,
       "judge": 62.756052141527014,
       "standard": 61.20211848749847
      }
     },
     "sealed|noul": {
      "cc": 22.806658654019163,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": -4.854368932038833,
       "judge": 36.8421052631579,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 42.919258845995714,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 62.11917736170213,
       "hard": 38.489070343137286,
       "judge": 48.63257153293411,
       "standard": 33.60970756345175
      }
     }
    }
   },
   "jevbench_score": 56.028575263593325,
   "key": "aplomb-1",
   "last_measured_on": "2026-10-06",
   "licence": "EmpirioLabs Model License 1.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "empiriolabsai/aplomb-1@f8e2dc81f0505db21a3b7a933ea479658e4ebb8f",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9333333333333333,
    "decisive_rate": 0.6818181818181818,
    "supported": true,
    "valid": 374
   },
   "open": "gated weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.21801638247211,
     "cost": 57.86508463809988,
     "intelligence": 45.81854920739271,
     "speed": 88.88310989631262
    },
    "board_release": "v1.6.43",
    "capability": 67.51828279493242,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 33,
    "capability_rank": 54,
    "composite_ci95": {
     "A": [
      40.003677562543594,
      66.59206455090654
     ],
     "B": [
      36.14823252368513,
      62.39027598829859
     ],
     "C": [
      27.780331640655273,
      46.3455307690425
     ]
    },
    "jevbench_score": 54.55838408944504,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 37,
    "rank": 61,
    "ranks": {
     "A": 61,
     "B": 64,
     "C": 69,
     "capability": 93
    },
    "scores": {
     "A": 54.55838408944504,
     "B": 50.34915295427923,
     "C": 37.88776672878128
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.33727095890790226,
     "p50_s_raw": 0.09363547945395112,
     "p95_s_adjusted": 0.3834511662367731,
     "p95_s_raw": 0.11672558311838657
    },
    "views": {
     "Balanced 33:33:33": 50.026426143216206,
     "Emphasis on Accuracy 60:20:20": 44.66300743458741,
     "Emphasis on Cost 20:20:60": 49.44235580546631,
     "Emphasis on Speed 20:60:20": 57.6274614302068,
     "Intelligence only": 38.475475184084495
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 117,
   "ranked": true,
   "ranks": {
    "A": 117,
    "B": 119,
    "C": 123,
    "capability": 119
   },
   "repo": "https://huggingface.co/empiriolabsai/aplomb-1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 56.028575263593325,
    "B": 50.81069771069998,
    "C": 38.90873282193981
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3351494163274765,
    "p50_s_raw": 0.09257470816373825,
    "p95_s_adjusted": 0.3977438145782798,
    "p95_s_raw": 0.12387190728913991
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.59491561004869,
      89.59945632360366
     ],
     "I": [
      39.712930281345386,
      49.434470191151824
     ],
     "capability": [
      62.49978696897133,
      68.99973905641266
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.94185474430773,
     "capability": 67.08763127147799,
     "intelligence": 45.23340779864823
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7173333333333334,
      "calibration": 85.7113652247,
      "cc": 63.85097051050084,
      "cc_by_type": {
       "choice": 63.85097051050084
      },
      "cc_tiered": 65.73882257352676,
      "cc_tiered_open": 70.2456274917742,
      "cc_tiered_sealed": 61.23201765527931,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 63.85097051050084
     },
     "noul": {
      "accuracy": 0.6346666666666667,
      "calibration": 90.41611374384358,
      "cc": 26.933333333333344,
      "cc_by_type": {
       "noul": 26.933333333333344
      },
      "cc_tiered": 24.791873150336016,
      "cc_tiered_open": 26.777087646652866,
      "cc_tiered_sealed": 22.806658654019163,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 26.933333333333344
     },
     "score": {
      "accuracy": 0.608,
      "calibration": 90.69808526437963,
      "cc": 43.92186809298247,
      "cc_by_type": {
       "score": 43.92186809298247
      },
      "cc_tiered": 45.16952767208192,
      "cc_tiered_open": 47.419796498168125,
      "cc_tiered_sealed": 42.919258845995714,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.92186809298247
     }
    },
    "raw_on_A": {
     "calibration": 89.64105266703848,
     "capability": 70.52705142972752,
     "intelligence": 51.413050192416556
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0040000000000000036,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 52.034185819368574,
    "Emphasis on Accuracy 60:20:20": 44.77114673772596,
    "Emphasis on Cost 20:20:60": 54.70209970013799,
    "Emphasis on Speed 20:60:20": 58.693172679006786,
    "Intelligence only": 37.02012791388867
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "APUS AI-LAB (@cryptoSUN2049 / findoo2049@gmail.com)",
   "axes": {
    "calibration": 67.91594971090585,
    "cost": null,
    "intelligence": 52.808647095759525,
    "speed": 80.338990145866
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.21087790603358122,
      "ece_score": 57.82441879328375,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 57.82441879328375,
      "score_before_time_limit": 57.82441879328375,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.0835579514824798,
      "brier": 0.14666984115552173,
      "ece": 0.13630441067160756,
      "n": 371,
      "score": 72.73911786567848,
      "score_before_time_limit": 72.73911786567848,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8238482384823849,
      "interval90_width": 0.22018970189701897,
      "n": 369,
      "nrps": 0.10709067085641949,
      "score": 73.18431247375533,
      "score_before_time_limit": 73.18431247375533,
      "time_limit_failed_share": 0,
      "top_ece": 0.21461153983423703
     }
    },
    "score": 67.91594971090585
   },
   "capability": 60.36229840333269,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED, and deliberately not invented. pricing_v15.reference() RAISES on the exact declared base Qwen/Qwen3.5-35B-A3B ('base model lacks an explicit market-listing decision'). Run 46's bongard-mini rule applies unchanged: the frozen module raises rather than estimating, and the frozen snapshot requires an explicit base-model listing under the published estimate rule. Complete but unpriced - shown, not ranked. PRICING REFERENCE NOTE: the dated snapshot ALREADY lists the exact base as qwen/qwen3.5-35b-a3b at (0.3125, 1.25), and BASE_REFERENCES ALREADY carries 'Qwen/Qwen3.5-35B-A3B-Base' -> 'qwen/qwen3.5-35b-a3b'. Only the non--Base (instruct) spelling of the same model is missing, so every Qwen3.5-35B-A3B derivative raises today. BECAUSE OUTPUT TOKENS ARE 0 BY CONSTRUCTION this row's Cost depends on the input rate alone, so the number is one multiplication over tokens the raw keeps; r56/receipts/COST-ARITHMETIC-R56.json carries it already computed.",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "APUS-OpenJev-v1-35B-A3B (merged bf16 checkpoint-5949 MoE on Qwen3.5-35B-A3B, their own native runtime, full depth, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights apus-ailab/APUS-OpenJev-v1-35B-A3B@090388abe2237117cc3b4df6f4250e81cbb36864, Apache-2.0, 66.99 GiB bf16 merged weights, checkpoint-5949, Qwen3_5MoeForConditionalGeneration (40 layers, 256 experts, 8 active), declared base Qwen/Qwen3.5-35B-A3B, full depth 40 layers (their depth_config declares exit_depth 20 / full_depth 40). accelerate>=1.10 additionally, per this repo's own requirements.txt. STACK: the authors' own requirements.txt, unmodified - torch==2.8.0 and transformers==5.16.1. The transformers pin is THEIRS, not a workaround: OpenJet.__init__ raises 'Layer execution is audited for transformers==5.16.1' on any other version. Served behind a TRANSPORT SHIM ONLY (r56/serve/apus_systemone_r56.py): loaded with THEIR OpenJet.from_pretrained(device=cuda:0, dtype=bfloat16) and every request handed to THEIR decide(), returning THEIR format_response() output. No contributor file edited. effort='high', the full-depth path, their decide() default and the configuration their own headline Frozen80 number is quoted on; effort='low' is not measured and no routing between the two is invented. One H100 PCIe 80 GB (Lium). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. MAPPING (run 56's, not the authors'; pre-registered in PREREG-R56 s2.2 before any item was sent and repeated here because it is part of the number): their runtime accepts primitive in {choice, noul, score_level} with 2-16 candidates labelled A-P. choice -> their 'choice' primitive, candidates = the item's own labels as id and its own criteria[label] as description, in labels order; their format_response returns probabilities keyed by candidate id, i.e. by the JevBench label already, so nothing is rewritten. All 750 choice items carry 3-9 options, inside their 2-16 window. noul -> their 'noul' primitive; their contract REQUIRES the canonical yes/no criteria there, so the item's own true/false description strings are NOT delivered - the proposition itself travels verbatim in instructions, which is what their noul is built to judge. score -> their 'choice' primitive over the ordinal levels, because their own RUNTIME.md rules score_level out as an ordinal Score API ('score_level is the judgement of a single proposition, it cannot be treated as a complete ordinal Score API'); the 4-5 ordered levels become candidates with id = the level index and description = the item's own level text, which carries the order. Recording all 375 score items as misses instead would understate a system whose published interface answers them. CALIBRATION IS MEASURED AS THE AUTHORS SHIP IT AND THEY DISCLOSED IT IN ADVANCE: their response carries calibrated: False and both their card and RUNTIME.md state the probabilities are an uncalibrated relative softmax over the offered candidates. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, NOT BY ESTIMATE: decide() runs one forward pass with logits_to_keep=1 and reads candidate logits; nothing is generated anywhere in that path. Input tokens are their own prompt_tokens field. THE 23 LONG ITEMS: their compile() raises ValueError('input exceeds runtime limit; no truncation permitted') above 8,192 tokens and their RUNTIME.md states it in advance. Recorded invalid. This is the first row in the series whose refusal on those items is guaranteed in the author's own code with an explicit no-truncation promise. add-requests run 56, GitHub #105, benchmarkheaven.com/submit ref AE709BBB",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100 PCIe",
   "intelligence": {
    "I_open": 54.71860410185919,
    "I_sealed": 50.898690089659866,
    "base": 52.808647095759525,
    "excess": 1.2465263697610829,
    "gap": 3.8199140121993267,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 60.14590221055973,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 54.70273670965714,
       "judge": 60.44825313117996,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 64.24775707384403,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 50,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 39.762153021173845,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.86605733086419,
       "hard": 29.028440368940068,
       "judge": 66.44632488030888,
       "standard": 15.151368382093667
      }
     },
     "sealed|choice": {
      "cc": 59.02771188966037,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 70.73387037173899,
       "hard": 51.131221719457,
       "judge": 59.031657355679705,
       "standard": 68.96169478999877
      }
     },
     "sealed|noul": {
      "cc": 49.333250807861994,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 32.038834951456316,
       "judge": 57.89473684210527,
       "standard": 57.746478873239425
      }
     },
     "sealed|score": {
      "cc": 44.33510757145725,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 70.35456392472854,
       "hard": 33.80629692129137,
       "judge": 52.81314502924963,
       "standard": 39.66594450846479
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "apus-openjev-v1-35b-a3b",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "listed",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "apus-ailab/APUS-OpenJev-v1-35B-A3B@090388abe2237117cc3b4df6f4250e81cbb36864",
   "not_ranked_because": "registry ranked=false or no composite",
   "noul_decisive": {
    "acc_among_decisive": 0.8529411764705882,
    "decisive_rate": 0.9164420485175202,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 67.91594971090585,
     "cost": null,
     "intelligence": 52.808647095759525,
     "speed": 80.50610262660216
    },
    "board_release": "v1.6.43",
    "capability": 60.36229840333269,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "capability": 66
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.859432493080385,
     "p50_s_raw": 0.3547162465401925,
     "p95_s_adjusted": 1.0355664409697056,
     "p95_s_raw": 0.4427832204848528
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": 52.808647095759525
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "capability": 66
   },
   "repo": "https://huggingface.co/apus-ailab/APUS-OpenJev-v1-35B-A3B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.8697583910310641,
    "p50_s_raw": 0.35987919551553205,
    "p95_s_adjusted": 1.0634139621281065,
    "p95_s_raw": 0.4567069810640532
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-35B-A3B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      63.096906433455544,
      71.75588820395149
     ],
     "I": [
      45.94798828612946,
      57.24766533646037
     ],
     "capability": [
      55.374556366190006,
      63.9661711269007
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 67.91594971090585,
     "capability": 60.36229840333269,
     "intelligence": 52.808647095759525
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6866666666666666,
      "calibration": 57.82441879328375,
      "cc": 59.92914183947025,
      "cc_by_type": {
       "choice": 59.92914183947025
      },
      "cc_tiered": 59.58680705011005,
      "cc_tiered_open": 60.14590221055973,
      "cc_tiered_sealed": 59.02771188966037,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.92914183947025
     },
     "noul": {
      "accuracy": 0.7733333333333333,
      "calibration": 72.73911786567848,
      "cc": 54.666666666666664,
      "cc_by_type": {
       "noul": 54.666666666666664
      },
      "cc_tiered": 56.790503940853014,
      "cc_tiered_open": 64.24775707384403,
      "cc_tiered_sealed": 49.333250807861994,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 54.666666666666664
     },
     "score": {
      "accuracy": 0.5946666666666667,
      "calibration": 73.18431247375533,
      "cc": 43.44124606258307,
      "cc_by_type": {
       "score": 43.44124606258307
      },
      "cc_tiered": 42.04863029631555,
      "cc_tiered_open": 39.762153021173845,
      "cc_tiered_sealed": 44.33510757145725,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.44124606258307
     }
    },
    "raw_on_A": {
     "calibration": 68.85867558536114,
     "capability": 63.06963064948997,
     "intelligence": 57.280585713618805
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": 52.808647095759525
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "APUS AI-LAB (@cryptoSUN2049 / findoo2049@gmail.com)",
   "axes": {
    "calibration": 66.61054590286017,
    "cost": 68.40093916429564,
    "intelligence": 47.20514429531085,
    "speed": 82.0651268880946
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.21593843550946115,
      "ece_score": 56.81231289810778,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 56.81231289810778,
      "score_before_time_limit": 56.81231289810778,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.1185983827493261,
      "brier": 0.1586174454486672,
      "ece": 0.14320810756172592,
      "n": 371,
      "score": 71.35837848765482,
      "score_before_time_limit": 71.35837848765482,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8401084010840109,
      "interval90_width": 0.29200542005420044,
      "n": 369,
      "nrps": 0.12208457963241476,
      "score": 71.66094632281795,
      "score_before_time_limit": 71.66094632281795,
      "time_limit_failed_share": 0,
      "top_ece": 0.22234824695561312
     }
    },
    "score": 66.61054590286017
   },
   "capability": 56.907845099085506,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 755,283 input + 0 output tokens over 1,477 decisions = 511.362897766 tokens/decision. GPU-time cost USD 0.0113058189868536/1,000; old list-price cost USD 0.0511362897765741/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0113058189868536/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.0503522
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011305818986853587
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "APUS-OpenJev-v1-9B (merged bf16 checkpoint-3000 on Qwen3.5-9B, their own native runtime, full depth, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights apus-ailab/APUS-OpenJev-v1-9B@82c9c56cfa9de8d36704ed91948d4726ef111635, Apache-2.0, 17.55 GiB bf16 merged weights, checkpoint-3000, declared base Qwen/Qwen3.5-9B, full depth 32 layers. STACK: the authors' own requirements.txt, unmodified - torch==2.8.0 and transformers==5.16.1. The transformers pin is THEIRS, not a workaround: OpenJet.__init__ raises 'Layer execution is audited for transformers==5.16.1' on any other version. Served behind a TRANSPORT SHIM ONLY (r56/serve/apus_systemone_r56.py): loaded with THEIR OpenJet.from_pretrained(device=cuda:0, dtype=bfloat16) and every request handed to THEIR decide(), returning THEIR format_response() output. No contributor file edited. effort='high', the full-depth path, their decide() default and the configuration their own headline Frozen80 number is quoted on; effort='low' is not measured and no routing between the two is invented. One H100 PCIe 80 GB (Lium). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. MAPPING (run 56's, not the authors'; pre-registered in PREREG-R56 s2.2 before any item was sent and repeated here because it is part of the number): their runtime accepts primitive in {choice, noul, score_level} with 2-16 candidates labelled A-P. choice -> their 'choice' primitive, candidates = the item's own labels as id and its own criteria[label] as description, in labels order; their format_response returns probabilities keyed by candidate id, i.e. by the JevBench label already, so nothing is rewritten. All 750 choice items carry 3-9 options, inside their 2-16 window. noul -> their 'noul' primitive; their contract REQUIRES the canonical yes/no criteria there, so the item's own true/false description strings are NOT delivered - the proposition itself travels verbatim in instructions, which is what their noul is built to judge. score -> their 'choice' primitive over the ordinal levels, because their own RUNTIME.md rules score_level out as an ordinal Score API ('score_level is the judgement of a single proposition, it cannot be treated as a complete ordinal Score API'); the 4-5 ordered levels become candidates with id = the level index and description = the item's own level text, which carries the order. Recording all 375 score items as misses instead would understate a system whose published interface answers them. CALIBRATION IS MEASURED AS THE AUTHORS SHIP IT AND THEY DISCLOSED IT IN ADVANCE: their response carries calibrated: False and both their card and RUNTIME.md state the probabilities are an uncalibrated relative softmax over the offered candidates. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, NOT BY ESTIMATE: decide() runs one forward pass with logits_to_keep=1 and reads candidate logits; nothing is generated anywhere in that path. Input tokens are their own prompt_tokens field. THE 23 LONG ITEMS: their compile() raises ValueError('input exceeds runtime limit; no truncation permitted') above 8,192 tokens and their RUNTIME.md states it in advance. Recorded invalid. This is the first row in the series whose refusal on those items is guaranteed in the author's own code with an explicit no-truncation promise. add-requests run 56, GitHub #106, benchmarkheaven.com/submit ref 2033065A",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100 PCIe",
   "intelligence": {
    "I_open": 52.06268273300428,
    "I_sealed": 42.347605857617424,
    "base": 47.20514429531085,
    "excess": 7.141689232948613,
    "gap": 9.715076875386856,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 61.13339106992548,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 56.59012268008808,
       "judge": 48.58272907053395,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 59.49758454106281,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 60.00000000000001,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 35.55707258802457,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.48792439337757,
       "hard": 38.515870391630514,
       "judge": 34.91730621356864,
       "standard": 23.633700639820077
      }
     },
     "sealed|choice": {
      "cc": 56.01292563590547,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 64.08247727440693,
       "hard": 44.00452488687781,
       "judge": 59.96275605214154,
       "standard": 70.07020569035596
      }
     },
     "sealed|noul": {
      "cc": 39.873501407011304,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72,
       "hard": 20.388349514563096,
       "judge": 39.47368421052631,
       "standard": 63.38028169014085
      }
     },
     "sealed|score": {
      "cc": 31.156390529935493,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 50.76837620384379,
       "hard": 19.973469764155748,
       "judge": 28.021772929225364,
       "standard": 48.418165625606015
      }
     }
    }
   },
   "jevbench_score": 56.59069443023599,
   "key": "apus-openjev-v1-9b",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "apus-ailab/APUS-OpenJev-v1-9B@82c9c56cfa9de8d36704ed91948d4726ef111635",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8470948012232415,
    "decisive_rate": 0.8814016172506739,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 66.61054590286017,
     "cost": 48.738123569705756,
     "intelligence": 47.20514429531085,
     "speed": 82.18245466956745
    },
    "board_release": "v1.6.43",
    "capability": 56.907845099085506,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 74,
    "capability_rank": 100,
    "composite_ci95": {
     "A": [
      33.00331563170442,
      56.572404110680004
     ],
     "B": [
      30.747622136678878,
      54.687915864792735
     ],
     "C": [
      22.918969188683615,
      40.193299875459424
     ]
    },
    "jevbench_score": 49.17976892452783,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 41,
    "rank": 65,
    "ranks": {
     "A": 65,
     "B": 66,
     "C": 71,
     "capability": 154
    },
    "scores": {
     "A": 49.17976892452783,
     "B": 47.01556694203153,
     "C": 34.15261730869988
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.705569653166458,
     "p50_s_raw": 0.277784826583229,
     "p95_s_adjusted": 0.8574615204473958,
     "p95_s_raw": 0.35373076022369787
    },
    "views": {
     "Balanced 33:33:33": 47.16399064408527,
     "Emphasis on Accuracy 60:20:20": 44.00058844315432,
     "Emphasis on Cost 20:20:60": 44.61842645447928,
     "Emphasis on Speed 20:60:20": 54.145911082892475,
     "Intelligence only": 39.978415802185516
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 116,
   "ranked": true,
   "ranks": {
    "A": 116,
    "B": 115,
    "C": 121,
    "capability": 186
   },
   "repo": "https://huggingface.co/apus-ailab/APUS-OpenJev-v1-9B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 56.59069443023599,
    "B": 52.93813496558611,
    "C": 39.299093354330545
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.7130538010736928,
    "p50_s_raw": 0.28152690053684637,
    "p95_s_adjusted": 0.8716959172161296,
    "p95_s_raw": 0.3608479586080648
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-9B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      62.07882152511099,
      70.58159568750249
     ],
     "I": [
      39.88317778412442,
      50.523530007279774
     ],
     "capability": [
      51.79741923709466,
      59.84489784641873
     ],
     "composite": null,
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 66.61054590286017,
     "capability": 56.907845099085506,
     "intelligence": 47.20514429531085
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6666666666666666,
      "calibration": 56.81231289810778,
      "cc": 57.37142748879814,
      "cc_by_type": {
       "choice": 57.37142748879814
      },
      "cc_tiered": 58.57315835291547,
      "cc_tiered_open": 61.13339106992548,
      "cc_tiered_sealed": 56.01292563590547,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.37142748879814
     },
     "noul": {
      "accuracy": 0.7386666666666667,
      "calibration": 71.35837848765482,
      "cc": 47.73333333333334,
      "cc_by_type": {
       "noul": 47.73333333333334
      },
      "cc_tiered": 49.685542974037055,
      "cc_tiered_open": 59.49758454106281,
      "cc_tiered_sealed": 39.873501407011304,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 47.73333333333334
     },
     "score": {
      "accuracy": 0.5306666666666666,
      "calibration": 71.66094632281795,
      "cc": 33.347875357101564,
      "cc_by_type": {
       "score": 33.347875357101564
      },
      "cc_tiered": 33.35673155898003,
      "cc_tiered_open": 35.55707258802457,
      "cc_tiered_sealed": 31.156390529935493,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 33.347875357101564
     }
    },
    "raw_on_A": {
     "calibration": 70.50531100654376,
     "capability": 61.66232173095365,
     "intelligence": 52.819332455363536
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 55.72061095835446,
    "Emphasis on Accuracy 60:20:20": 49.32241623650303,
    "Emphasis on Cost 20:20:60": 57.70724820358342,
    "Emphasis on Speed 20:60:20": 61.589860603447896,
    "Intelligence only": 42.07537349920102
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Arcbright (arcbright on Hugging Face)",
   "axes": {
    "calibration": 86.16803253832052,
    "cost": 67.83160602523185,
    "intelligence": 43.75476217706713,
    "speed": 91.60645552542691
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06217474017608143,
      "ece_score": 87.56505196478372,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 87.56505196478372,
      "score_before_time_limit": 87.56505196478372,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3672922252010724,
      "brier": 0.11447032458100269,
      "ece": 0.07345892216986336,
      "n": 373,
      "score": 85.30821556602733,
      "score_before_time_limit": 85.30821556602733,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9516129032258065,
      "interval90_width": 0.4168906810035842,
      "n": 372,
      "nrps": 0.09282356443339915,
      "score": 85.63083008415052,
      "score_before_time_limit": 85.63083008415052,
      "time_limit_failed_share": 0,
      "top_ece": 0.09727991694179526
     }
    },
    "score": 86.16803253832052
   },
   "capability": 64.96139735769383,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 777,809 input + 1,485 output tokens over 1,485 decisions = 524.777104377 tokens/decision. GPU-time cost USD 0.0118108131565639/1,000; old list-price cost USD 0.0263388552188552/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0118108131565639/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.026075466666666672
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011810813156563906
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Arcbright Elf 1 12B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-20",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (compute capability 12.0, driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.477/h. The author's own `python3 -m elf_serve` from the pinned commit, ELF_MODEL pointed at the commit-pinned local snapshot, on 127.0.0.1:8000 inside a network-none Docker container with the cut proven from inside (loopback only, no default route, DNS and TCP fail), offline and credential-free, one request at a time. Timed over loopback on the VM. Measured 20 Oct 2026.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 49.76480591481253,
    "I_sealed": 37.74471843932173,
    "base": 43.75476217706713,
    "excess": 2.180560705135086,
    "gap": 12.020087475490797,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 69.65737475588615,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 69.80182447310474,
       "judge": 64.40342781806197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 28.84057971014493,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 30.434782608695656,
       "judge": 0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 50.796463278406506,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.67881522863554,
       "hard": 58.047006362950924,
       "judge": 42.020712273591265,
       "standard": 40.517827641425995
      }
     },
     "sealed|choice": {
      "cc": 63.52389731980814,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 84.75222363405337,
       "hard": 57.84665201804093,
       "judge": 61.21449559255633,
       "standard": 67.72832735709765
      }
     },
     "sealed|noul": {
      "cc": 14.103279119233829,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 41.37931034482758,
       "hard": 7.272727272727275,
       "judge": 4.615384615384621,
       "standard": 28.358208955223873
      }
     },
     "sealed|score": {
      "cc": 35.60697887892322,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 54.33201940856638,
       "hard": 26.4903029591115,
       "judge": 36.034132628127544,
       "standard": 43.8370798299186
      }
     }
    }
   },
   "jevbench_score": 50.95211596400614,
   "key": "arcbright-elf-1-12b",
   "last_measured_on": "2026-10-20",
   "licence": "Apache-2.0 (model card, root LICENSE and serve/LICENSE; base google/gemma-4-12B-it under the Gemma 4 licence), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.34",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "arcbright/elf-1-12b@da29a170016f3a179821296bcc8221ac222a44c6",
   "noul_decisive": {
    "acc_among_decisive": 0.9661016949152542,
    "decisive": 236,
    "decisive_rate": 0.6327077747989276,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.16803253832052,
     "cost": 57.38209314881071,
     "intelligence": 43.75476217706713,
     "speed": 91.97994026431576
    },
    "board_release": "v1.6.43",
    "capability": 64.96139735769383,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 38,
    "capability_rank": 60,
    "composite_ci95": {
     "A": [
      34.34154919677984,
      62.01915333142578
     ],
     "B": [
      30.729584588233603,
      57.89694395385064
     ],
     "C": [
      23.848298053319336,
      43.0688564801568
     ]
    },
    "jevbench_score": 48.80851298722302,
    "measured_in": "v1.6.34",
    "method": "v1.6",
    "open_board_rank": 42,
    "rank": 66,
    "ranks": {
     "A": 66,
     "B": 68,
     "C": 72,
     "capability": 102
    },
    "scores": {
     "A": 48.80851298722302,
     "B": 44.723760281190714,
     "C": 33.894800685571546
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21816718400006038,
     "p50_s_raw": 0.034083592000030194,
     "p95_s_adjusted": 0.2905471016999911,
     "p95_s_raw": 0.07027355084999556
    },
    "views": {
     "Balanced 33:33:33": 44.91128082306797,
     "Emphasis on Accuracy 60:20:20": 39.529652968308206,
     "Emphasis on Cost 20:20:60": 44.51878166820759,
     "Emphasis on Speed 20:60:20": 52.52523519227407,
     "Intelligence only": 33.50703306610661
    }
   },
   "rank": 121,
   "ranked": true,
   "ranks": {
    "A": 121,
    "B": 122,
    "C": 127,
    "capability": 130
   },
   "repo": "https://huggingface.co/arcbright/elf-1-12b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 50.95211596400614,
    "B": 46.14693327380985,
    "C": 35.38341386389315
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23457195800002636,
    "p50_s_raw": 0.042285979000013185,
    "p95_s_adjusted": 0.2944952907999777,
    "p95_s_raw": 0.07224764539998886
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 53.221149190341464,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": -15.336552428975603
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 53.221149190341464
      },
      "calibration_gen": {
       "accuracy": 0.5544554455445545,
       "cc": 34.513264583851004,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": 28.000000000000004,
        "score": 35.985180814119275
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 34.513264583851004
      },
      "composite_gen": {
       "accuracy": 0.6336633663366337,
       "cc": 32.83917641063007,
       "cc_by_type": {
        "choice": 70.73170731707317,
        "noul": -18.518518518518523,
        "score": 46.30434043333555
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 32.83917641063007
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 79.39112055434624,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 78.17336166303872
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 79.39112055434624
      },
      "jaggedness": {
       "accuracy": 0.4888888888888889,
       "cc": 9.134700657058808,
       "cc_by_type": {
        "choice": 49.4706448508181,
        "noul": -45.945945945945944,
        "score": 23.87940306630427
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 9.134700657058808
      },
      "llm_router": {
       "accuracy": 0.8083067092651757,
       "cc": 68.30021817589943,
       "cc_by_type": {
        "choice": 75.43365325350004,
        "noul": 77.31958762886597,
        "score": 52.14741364533227
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.30021817589943
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5185185185185185,
       "cc": 9.266765553937715,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 0,
        "score": -22.715167255712633
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 9.266765553937715
      },
      "multilingual": {
       "accuracy": 0.6654545454545454,
       "cc": 44.16630479379578,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": 13.888888888888884,
        "score": 52.84543468715122
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 44.16630479379578
      },
      "negation_pair_gen": {
       "accuracy": 0.5,
       "cc": 4.563370710907104,
       "cc_by_type": {
        "choice": 63.333333333333336,
        "noul": -53.333333333333336,
        "score": 3.6901121327213127
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 4.563370710907104
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 72.7892309873725,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100,
        "score": 42.391716986141546
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 72.7892309873725
      },
      "usecase": {
       "accuracy": 0.5909090909090909,
       "cc": 37.411840669893564,
       "cc_by_type": {
        "choice": 62.00980392156864,
        "noul": 12.5,
        "score": 37.725718088112046
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 37.411840669893564
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": -2.2963023047605,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 0,
        "score": -68.7936688190434
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 21.105620677411725,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 17.387522554508593
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 21.105620677411725
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 78.60884686810552,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100,
        "score": 75.64954945387412
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 78.60884686810552
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 33.20380808987042,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 99.79764400890363
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 33.20380808987042
      },
      "en": {
       "accuracy": 0.6550825369244135,
       "cc": 41.21871964823898,
       "cc_by_type": {
        "choice": 65.5117706642584,
        "noul": 19.424460431654666,
        "score": 38.71992784880387
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 41.21871964823898
      },
      "es": {
       "accuracy": 0.6774193548387096,
       "cc": 52.57126625614512,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 75,
        "score": 2.289655212154784
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 52.57126625614512
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 55.16848435692223,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 97.74589022923665
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 55.16848435692223
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 76.73155887991228,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 72.54761781620745
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 76.73155887991228
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 41.739169283908474,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 50,
        "score": -8.022715611961729
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 41.739169283908474
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 65.060016125563,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 23.23928454192673
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 65.060016125563
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 30.419424479507466,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 28.446518077124473
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 30.419424479507466
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 8.69703083755734,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 26.09109251267202
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 8.69703083755734
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.65244468093822,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.62400070948131
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.65244468093822
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 2.0314007863726835,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": -8.62441235949667
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 2.0314007863726835
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 56.852101069094736,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": 39.99999999999999,
        "score": 61.02604506086019
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 56.852101069094736
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 60.2311405918991,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 63.121235564736764
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 60.2311405918991
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 7.854063750825509,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 20.157935933327575
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 7.854063750825509
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 8.62512772013944,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": -30.646355970016458
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 8.62512772013944
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 68.40627899393645,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 69.73496601406745
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 68.40627899393645
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 38.58243816800493,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 49.080647837348124
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 38.58243816800493
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7333333333333333,
       "cc": 65.55617214259745,
       "cc_by_type": {
        "choice": 65.55617214259745
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 65.55617214259745
      },
      "noul": {
       "accuracy": 0.608,
       "cc": 21.599999999999998,
       "cc_by_type": {
        "noul": 21.599999999999998
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 21.599999999999998
      },
      "score": {
       "accuracy": 0.56,
       "cc": 39.95424165202871,
       "cc_by_type": {
        "score": 39.95424165202871
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 39.95424165202871
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -15.780619190966505,
       "cc_by_type": {
        "choice": 36,
        "noul": -75,
        "score": -8.341857572899514
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.6666666666666666,
       "cc": 43.202215257321164,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 55.55555555555556,
        "score": 11.31817096174953
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 43.202215257321164
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 26.38194646652667,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": 11.66616460283204
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 26.38194646652667
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -35.273927478672455,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -35.64634383952614
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 30.238339740416034,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 53.033859800958226
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 30.238339740416034
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -23.83827556753779,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -33.333333333333336,
        "score": -87.86702796047499
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 40.969829317719324,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 38.03553837332603
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 40.969829317719324
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": 0.870256604504983,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": -100,
        "score": 44.42261650341042
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.870256604504983
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 11.172183319917124,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -19.999999999999996,
        "score": 16.01654995975137
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 11.172183319917124
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 39.61960806069081,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 16.431986209195905
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 39.61960806069081
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 34.1684274583135,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 11.111111111111116,
        "score": 14.649985217317763
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 34.1684274583135
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -33.228079786697315,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -35.168110327833865
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 66.36226749196166,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 50.6031172551556
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 66.36226749196166
      },
      "model_routing": {
       "accuracy": 0.8202764976958525,
       "cc": 70.15069627663628,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 90.90909090909092,
        "score": 39.413127790947776
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 70.15069627663628
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 10.565354791083555,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 66.24500871106257
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 10.565354791083555
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 48.01941397788096,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 44.151092444320696
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 48.01941397788096
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 28.233025996560656,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0,
        "score": 29.0340533591401
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 28.233025996560656
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 5.217071352811868,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": -33.333333333333336,
        "score": 35.94106913089937
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 5.217071352811868
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 95.3686978433633,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 96.7822501137198
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 95.3686978433633
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.62303224336958,
      87.64236723127183
     ],
     "I": [
      37.79040779205359,
      48.44381950020701
     ],
     "capability": [
      60.23623630844356,
      67.39815359481638
     ],
     "composite": null,
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.16803253832052,
     "capability": 64.96139735769383,
     "intelligence": 43.75476217706713
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7333333333333333,
      "calibration": 87.56505196478372,
      "cc": 65.55617214259745,
      "cc_by_type": {
       "choice": 65.55617214259745
      },
      "cc_tiered": 66.59063603784715,
      "cc_tiered_open": 69.65737475588615,
      "cc_tiered_sealed": 63.52389731980814,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.55617214259745
     },
     "noul": {
      "accuracy": 0.608,
      "calibration": 85.30821556602733,
      "cc": 21.599999999999998,
      "cc_by_type": {
       "noul": 21.599999999999998
      },
      "cc_tiered": 21.471929414689377,
      "cc_tiered_open": 28.84057971014493,
      "cc_tiered_sealed": 14.103279119233829,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 21.599999999999998
     },
     "score": {
      "accuracy": 0.56,
      "calibration": 85.63083008415052,
      "cc": 39.95424165202871,
      "cc_by_type": {
       "score": 39.95424165202871
      },
      "cc_tiered": 43.20172107866486,
      "cc_tiered_open": 50.796463278406506,
      "cc_tiered_sealed": 35.60697887892322,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 39.95424165202871
     }
    },
    "raw_on_A": {
     "calibration": 84.45522192019058,
     "capability": 62.52355684051309,
     "intelligence": 40.591891760835594
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 47.35556532216023,
    "Emphasis on Accuracy 60:20:20": 40.63736029631815,
    "Emphasis on Cost 20:20:60": 49.09041469233688,
    "Emphasis on Speed 20:60:20": 54.43049688417504,
    "Intelligence only": 33.50703306610661
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Arcbright (arcbright on Hugging Face)",
   "axes": {
    "calibration": 92.92233551579638,
    "cost": 71.73785397462883,
    "intelligence": 66.79530416169641,
    "speed": 92.52857247879322
   },
   "base_model": "google/gemma-4-12B-it",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.043603442006377814,
      "ece_score": 91.27931159872443,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 354,
      "n_tvd": 0,
      "score": 91.27931159872443,
      "score_before_time_limit": 91.27931159872443,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.08274802736329191,
      "ece": 0.02950437714957693,
      "n": 375,
      "score": 94.09912457008461,
      "score_before_time_limit": 94.09912457008461,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9653333333333334,
      "interval90_width": 0.267111111111111,
      "n": 375,
      "nrps": 0.05289406083236051,
      "score": 93.38857037858008,
      "score_before_time_limit": 93.38857037858008,
      "time_limit_failed_share": 0,
      "top_ece": 0.03966726579801887
     }
    },
    "score": 92.92233551579638
   },
   "capability": 79.8588198387464,
   "code_revision": "The author's own `elf_serve` package 1.1.0 at the pinned revision 1018696022ae3fdd9cb8a93e3e972f69c11184c1 (the Hub main and the submitted revision), unmodified, with ONE documented non-default option, ELF_TRUNCATE=1 (the shipped default refuses an over-long row): `python3 -m elf_serve` with ELF_MODEL pointing at the commit-pinned local snapshot, ELF_HOST=127.0.0.1, PORT=8000, no ELF_CONFIG, no engine override, no API key. The served configuration is the shipped decision_config.json (sha256 43710e1524b56c8dd8a71d5101301cf3ac87d36013c3c9bcd051d4173e7a7f57): format hybrid-v1, bf16, max_model_len 40,960, gpu_memory_utilization 0.9, temperatures 1.13 scalar / choice 1.2391 / noul 1.13 / score 0.9234, noul_commit [0.199, 0.801], joint 'generic' (inert at one question per request). The scored process's /health after the run reported hf_revision equal to the pin, package_version 1.1.0, truncate true, noul_commit [0.199, 0.801], the config hash above and an alive engine. Ran on the stock digest-pinned vllm/vllm-openai:v0.30.0 image (sha256 8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90) plus four hash-pinned wheels at the author's tested versions (transformers 5.17.0, huggingface-hub 1.33.0, fastapi 0.136.3, uvicorn 0.54.0, offline --no-deps install); the author's serve package installed from files verified against the author's own serve/SHA256SUMS (11 of 11 OK). Frozen harness: the frozen v1.6 measurement driver with the frozen TypeSafe adapter, one request at a time, no response shim. Independent source review of the delta from the previously measured revision (line by line): PASS with conditions, no blocker. Two audited code patterns ride along unchanged from the earlier revision (two read-only nvidia-smi subprocess queries with constant arguments in settings.py, and a snapshot_download fallback that is unreachable as measured because the local snapshot directory is passed and the container is network-none); both were disclosed and cleared for the earlier revision and ran here only on a disposable rented GPU VM. Weights: the commit-pinned snapshot, all 29 files verified by sha256 and git blob oid on the node (23,951,919,471 bytes). Asserted before any weights step: vllm 0.30.0, flashinfer-python 0.6.18.post1, torch 2.13.0, python 3.12, elf-serve 1.1.0; a first-party Triton gate (compiler, Python headers, libcuda and one compiled kernel) passed before the weights, in the smoke and in the run container. The measurement kit was hash-pinned (bcbaca2d) and independently reviewed (PASS) before measurement.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 577,488 input + 1,489 output tokens over 1,489 decisions = 388.836131632 tokens/decision. GPU-time cost USD 0.00875127908767551/1,000; old list-price cost USD 0.0195418065815984/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00875127908767551/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.020511333333333333
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.008751279087675506
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "DRAW: measured on the 27 Oct 2026 regular draw (fresh sealed items; not yet equated to the reference scale, so its numbers are shown as published, offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own draw. SECOND ROW ON THESE WEIGHTS, A NEW REVISION: arcbright/elf-1-12b@10186960 is a new revision of the repository whose earlier revision da29a170 is the live Arcbright Elf 1 12B row. All five safetensors shards, the index, config, tokenizer files, chat template and render.py are byte-identical to the measured earlier revision (by sha256 and git blob oid); the new revision changes only decision_config.json (adds noul_commit [0.199, 0.801]), the serve code (serve/elf_serve/{__init__,server,settings}.py and a new truncate.py), the version and the READMEs. It is a separate row and replaces nothing; the earlier row stays on the board unchanged. The name is 'Jev-compatible' only: it is an Elf model, not a Jev model. AUTHOR'S OWN SERVER on our rented hardware, run unmodified apart from the documented option ELF_TRUNCATE=1 (see code_revision), no shim. NATIVE OUTPUTS: probabilities for choice, noul and score from the per-marker logit readout of one vLLM forward pass, at the shipped per-type temperatures. TRUNCATION (author's documented option ELF_TRUNCATE=1, a non-default setting we enabled so that long items are answered): a row of 40,960 tokens or more is re-rendered with its state cut to the first and last 3,000 tokens joined by ' … '. Exactly the 5 pre-registered long items were cut and answered with HTTP 200 (30,789 input tokens in total, about 6,160 each); their answers rest on the first and last 3,000 state tokens, not the whole state. They are answered, not refused, and counted as answers. All other rows are unchanged. COMMIT BAND (noul_commit [0.199, 0.801], new in this revision, disclosed like the DOPEST floor): yes/no probabilities strictly inside (0.199, 0.801) are moved to the nearer band edge; the argmax is unchanged. The band equals the abstention band JevBench counts; the author does not name JevBench, but the effect is benchmark-aware by construction. The Calibration axis includes this transform and we did not measure a variant without it. JOINT MODE: decision_config.json ships joint 'generic'; with exactly one question per request it is a pass-through, so the row is the author-default configuration apart from ELF_TRUNCATE=1. TRAINING DATA, the author's own statement: fine-tuned from google/gemma-4-12B-it on 84,470 rows including ZefanCai/Open-Jev and Open-Jev-v1.1 (public Open-Jev material) and the train splits of ten Decision Index benchmarks; the author states it screened for overlap with DI test items - not independently verified here, and overlap with JevBench items was not checked, so contamination of the sealed draw is unchecked. SELF-REPORTED, unverified: DI public index 58.35 and 54.5 ms median latency on the author's card. English only; not evaluated for decisions about people. LoRA on attention and MLP merged into bf16 (11,959,730,176 parameters). LICENCE NOTE: Apache-2.0 on the weights and the serve code; the base is Gemma 4 under its own terms. ANSWERED: 1,500 of 1,500, HTTP 200 on every row, no refusal, no error; none over 10 s (maximum 0.557 s; measured p50 0.029 s, p95 0.060 s). USAGE: server-native input usage totals 610,840 tokens and equals our first-party count exactly, 0 mismatches; output_tokens 1 per answered row (1,500 in total). DETERMINISM (record-only): the first 50 requests were run twice; 1 of 50 was bit-identical, 0 argmax flips, maximum probability difference 0.0327 (serial vLLM path; cause not isolated). The scored rows are the first run and stand. The scored loop of 1,500 requests took 55 s. ATTEMPT HISTORY: one attempt on this draw's sealed input, one create call that produced the only instance; the pre-sealed smoke (including the cut probes), network-cut proof, pre-warm and TTL fit gate passed before any sealed item. HARDWARE: one Verda RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, EU); latency reflects that card.",
   "display": "Arcbright Elf 1 12B (r2)",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-27",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.14",
    "results_sha256": "9e460d48571eeb00d94f554a985fdc2ebd52b63beb9aed7a4efe88762eb7b32e",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB, compute capability 12.0, driver 580.178.04), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, image 24.04.cuda13.0, USD 2.608 per hour recorded by the runner, 150 GB disk. One VM for this row, one attempt. Measured 27 Oct 2026, VM window about 17:54-18:06Z, about USD 0.5 GPU rental (elapsed time at the recorded hourly rate). Author server at 127.0.0.1:8000 inside a network-none Docker container (cap-drop ALL), network cut proven from inside (loopback only, no default route, DNS and TCP fail), offline and credential-free, one request at a time, private output. Before sealed input: model files verified, the Triton gate, the smoke and a pre-warm that paid Triton's per-shape compilation passed, and the TTL fit gate admitted the run. Input sha256 9b2f32edaccb29afd726af8cd54cf1a22d85e8b1824566924045d83c6f828426 (v1.6-regular-20261027) was re-checked inside the container (match). Complete 1,500-row run, rc 0. Raw output sha256 84f08ebb6ff92af453b75cda042f617ea89c5be6a2176c7c69e11f2521da6f08 (the pull-hash receipt). Node-shred and local-shred receipts verified; exact-id teardown receipt confirms the VM closed. GPU rental is separate from the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 63.12196868957201,
    "I_sealed": 70.4686396338208,
    "base": 66.79530416169641,
    "excess": -9.920058586687036,
    "gap": -7.3466709442487925,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 71.16728353223091,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 73.57659641396666,
       "judge": 64.40342781806197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 67.42028985507247,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 65.21739130434783,
       "judge": 60.00000000000001,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 50.77833268141265,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.67881522863554,
       "hard": 58.047006362950924,
       "judge": 41.960277081046485,
       "standard": 40.51782744527389
      }
     },
     "sealed|choice": {
      "cc": 72.1654141738766,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 96.06943989518508,
       "hard": 56.73991126135643,
       "judge": 81.36324417601381,
       "standard": 77.26766213505685
      }
     },
     "sealed|noul": {
      "cc": 77.54822305239892,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 90.80459770114942,
       "hard": 60.97560975609757,
       "judge": 91.11111111111111,
       "standard": 83.72093023255813
      }
     },
     "sealed|score": {
      "cc": 61.69228167518689,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 90.67658303962506,
       "hard": 39.350389802771836,
       "judge": 71.88582039140796,
       "standard": 76.59360666346626
      }
     }
    }
   },
   "jevbench_score": 79.23945650494228,
   "key": "arcbright-elf-1-12b-r2",
   "last_measured_on": "2026-10-27",
   "licence": "Apache-2.0 (model card, root LICENSE and serve/LICENSE; base google/gemma-4-12B-it under the Gemma 4 licence), public, not gated",
   "listing": "unranked",
   "measured_in": "v1.7.14",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "arcbright/elf-1-12b@1018696022ae3fdd9cb8a93e3e972f69c11184c1",
   "new_in": "v1.7.14",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale: its first anchor runs used models weaker than this row, and stronger anchors are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.8933333333333333,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/arcbright/elf-1-12b",
   "scores": {
    "A": 79.23945650494228,
    "B": 76.39301013275472,
    "C": 79.23945650494228
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20765690400005496,
    "p50_s_raw": 0.028828452000027482,
    "p95_s_adjusted": 0.26902731509995875,
    "p95_s_raw": 0.05951365754997937
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.75,
       "cc": 66.66666666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 66.66666666666666
      },
      "calibration_gen": {
       "accuracy": 0.5714285714285714,
       "cc": 54.32279957021712,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 100,
        "score": 14.916450658703317
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 54.32279957021712
      },
      "composite_gen": {
       "accuracy": 0.8947368421052632,
       "cc": 84.29588128077988,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 52.88764384233965
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 84.29588128077988
      },
      "fairness": {
       "accuracy": 1,
       "cc": 98.61891372447077,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.85674117341235
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 98.61891372447077
      },
      "jaggedness": {
       "accuracy": 0.45454545454545453,
       "cc": 32.80015658686387,
       "cc_by_type": {
        "choice": 25.92592592592593,
        "noul": 33.33333333333333,
        "score": 39.141210501332345
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 32.80015658686387
      },
      "llm_router": {
       "accuracy": 0.8444444444444444,
       "cc": 78.4976854585687,
       "cc_by_type": {
        "choice": 84.13631022326675,
        "noul": 100,
        "score": 51.35674615243933
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 78.4976854585687
      },
      "long_state": {
       "accuracy": 0.6,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 67.74193548387098
      },
      "ml_usecase": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 100
      },
      "multilingual": {
       "accuracy": 0.8219326818675353,
       "cc": 73.53404416459962,
       "cc_by_type": {
        "choice": 72.51562836818282,
        "noul": 77.86885245901641,
        "score": 70.21765166659965
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 73.53404416459962
      },
      "negation_pair_gen": {
       "accuracy": 0.6818181818181818,
       "cc": 48.33333333333333,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": 50
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 48.33333333333333
      },
      "option_order": {
       "accuracy": 0.8888888888888888,
       "cc": 85.71428571428571,
       "cc_by_type": {
        "choice": 85.71428571428571
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 85.71428571428571
      },
      "perturb_pair": {
       "accuracy": 0.8928571428571429,
       "cc": 84.83408129900955,
       "cc_by_type": {
        "choice": 81.13207547169812,
        "noul": 100,
        "score": 73.37016842533058
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 84.83408129900955
      },
      "usecase": {
       "accuracy": 0.7661538461538462,
       "cc": 66.7120201786676,
       "cc_by_type": {
        "choice": 64.25966447848285,
        "noul": 75.38461538461539,
        "score": 60.49178067290457
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 66.7120201786676
      }
     },
     "lang": {
      "ar": {
       "accuracy": 1,
       "cc": 97.77486142695967,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 93.32458428087902
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 97.77486142695967
      },
      "cs": {
       "accuracy": 0.7567567567567568,
       "cc": 65.4600696972371,
       "cc_by_type": {
        "choice": 59.89304812834225,
        "noul": 77.77777777777777,
        "score": 58.70938318559129
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 65.4600696972371
      },
      "da": {
       "accuracy": 0.6944444444444444,
       "cc": 38.60489924499188,
       "cc_by_type": {
        "choice": 70.18633540372672,
        "noul": 27.27272727272727,
        "score": 18.355635058521656
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 38.60489924499188
      },
      "de": {
       "accuracy": 0.75,
       "cc": 68.00511327029649,
       "cc_by_type": {
        "choice": 67.58425520967327,
        "noul": 75,
        "score": 61.43108460121618
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 68.00511327029649
      },
      "el": {
       "accuracy": 0.75,
       "cc": 63.43152562645863,
       "cc_by_type": {
        "choice": 65.38461538461539,
        "noul": 39.99999999999999,
        "score": 84.9099614947605
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 63.43152562645863
      },
      "en": {
       "accuracy": 0.7837837837837838,
       "cc": 68.97224296083085,
       "cc_by_type": {
        "choice": 69.06035141329258,
        "noul": 78.68852459016394,
        "score": 59.167852879036076
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 68.97224296083085
      },
      "es": {
       "accuracy": 0.8108108108108109,
       "cc": 72.1649647075799,
       "cc_by_type": {
        "choice": 92.78267842822774,
        "noul": 77.77777777777777,
        "score": 45.93443791673416
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 72.1649647075799
      },
      "fi": {
       "accuracy": 0.8076923076923077,
       "cc": 73.93595591461282,
       "cc_by_type": {
        "choice": 65.71428571428571,
        "noul": 89.4736842105263,
        "score": 66.61989781902642
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 73.93595591461282
      },
      "fr": {
       "accuracy": 0.84375,
       "cc": 76.76005018292469,
       "cc_by_type": {
        "choice": 69.38775510204081,
        "noul": 100,
        "score": 60.892395446733246
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 76.76005018292469
      },
      "hi": {
       "accuracy": 0.7837837837837838,
       "cc": 68.08068944406175,
       "cc_by_type": {
        "choice": 66.94214876033057,
        "noul": 55.55555555555556,
        "score": 81.74436401629913
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 68.08068944406175
      },
      "id": {
       "accuracy": 0.8048780487804879,
       "cc": 67.47064898282078,
       "cc_by_type": {
        "choice": 63.13364055299538,
        "noul": 66.66666666666667,
        "score": 72.61163972880024
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 67.47064898282078
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 84.0153800280403,
       "cc_by_type": {
        "choice": 92.01419698314109,
        "noul": 66.66666666666667,
        "score": 93.36527643431313
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 84.0153800280403
      },
      "ja": {
       "accuracy": 0.8636363636363636,
       "cc": 74.51716996187086,
       "cc_by_type": {
        "choice": 61.139896373057,
        "noul": 81.81818181818181,
        "score": 80.59343169437376
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 74.51716996187086
      },
      "ko": {
       "accuracy": 0.7906976744186046,
       "cc": 74.75707256705293,
       "cc_by_type": {
        "choice": 77.94117647058825,
        "noul": 80,
        "score": 66.33004123057056
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 74.75707256705293
      },
      "mixed": {
       "accuracy": 0.859375,
       "cc": 76.96432363797665,
       "cc_by_type": {
        "choice": 78.3978397839784,
        "noul": 80,
        "score": 72.49513112995156
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 76.96432363797665
      },
      "nl": {
       "accuracy": 0.8611111111111112,
       "cc": 61.55569565225476,
       "cc_by_type": {
        "choice": 85.84905660377359,
        "noul": 80,
        "score": 18.81803035299069
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 61.55569565225476
      },
      "no": {
       "accuracy": 0.7843137254901961,
       "cc": 63.338139572024254,
       "cc_by_type": {
        "choice": 58.691910499139425,
        "noul": 86.66666666666667,
        "score": 44.65584155026667
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 63.338139572024254
      },
      "pl": {
       "accuracy": 0.8070175438596491,
       "cc": 73.4366995475141,
       "cc_by_type": {
        "choice": 66.47152740819585,
        "noul": 88.23529411764706,
        "score": 65.60327711669936
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 73.4366995475141
      },
      "pt": {
       "accuracy": 0.7872340425531915,
       "cc": 69.15606629284918,
       "cc_by_type": {
        "choice": 82.02995008319466,
        "noul": 66.66666666666667,
        "score": 58.77158212868616
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 69.15606629284918
      },
      "sv": {
       "accuracy": 0.8043478260869565,
       "cc": 73.14860318912982,
       "cc_by_type": {
        "choice": 59.776536312849174,
        "noul": 83.33333333333333,
        "score": 76.33593992120693
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 73.14860318912982
      },
      "tr": {
       "accuracy": 0.8478260869565217,
       "cc": 83.348503870194,
       "cc_by_type": {
        "choice": 65.79247434435577,
        "noul": 100,
        "score": 84.2530372662262
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 83.348503870194
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 78.44885023002021,
       "cc_by_type": {
        "choice": 72.6027397260274,
        "noul": 100,
        "score": 62.74381096403325
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 78.44885023002021
      },
      "zh": {
       "accuracy": 0.8780487804878049,
       "cc": 82.72762800492681,
       "cc_by_type": {
        "choice": 87.60330578512396,
        "noul": 100,
        "score": 60.57957822965647
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 82.72762800492681
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.788,
       "cc": 71.08243829000413,
       "cc_by_type": {
        "choice": 71.08243829000413
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 71.08243829000413
      },
      "noul": {
       "accuracy": 0.8933333333333333,
       "cc": 78.66666666666666,
       "cc_by_type": {
        "noul": 78.66666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 78.66666666666666
      },
      "score": {
       "accuracy": 0.7573333333333333,
       "cc": 65.88786028381091,
       "cc_by_type": {
        "score": 65.88786028381091
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 65.88786028381091
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7916666666666666,
       "cc": 78.26903429437168,
       "cc_by_type": {
        "choice": 64.41808747220162,
        "noul": 100,
        "score": 70.38901541091343
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 78.26903429437168
      },
      "code_linting": {
       "accuracy": 0.7681159420289855,
       "cc": 66.48432655484892,
       "cc_by_type": {
        "choice": 74.28046540110228,
        "noul": 66.66666666666667,
        "score": 58.50584759677784
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 66.48432655484892
      },
      "customer_support": {
       "accuracy": 0.8333333333333334,
       "cc": 64.01015718184654,
       "cc_by_type": {
        "choice": 85.239852398524,
        "noul": 19.999999999999996,
        "score": 86.7906191470156
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 64.01015718184654
      },
      "demand_forecasting": {
       "accuracy": 0.5303030303030303,
       "cc": 24.448263682335284,
       "cc_by_type": {
        "choice": 22.668688400303264,
        "noul": 23.076923076923084,
        "score": 27.59917956977951
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 24.448263682335284
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8387096774193549,
       "cc": 76.74863444855406,
       "cc_by_type": {
        "choice": 70.44334975369458,
        "noul": 76.47058823529412,
        "score": 83.33196535667349
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 76.74863444855406
      },
      "feature_extraction": {
       "accuracy": 0.8333333333333334,
       "cc": 79.31821700034652,
       "cc_by_type": {
        "choice": 66.08040201005025,
        "noul": 100,
        "score": 71.87424899098932
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 79.31821700034652
      },
      "financial_crime": {
       "accuracy": 0.8253968253968254,
       "cc": 62.182991610544796,
       "cc_by_type": {
        "choice": 80.86124401913877,
        "noul": 63.63636363636365,
        "score": 42.05136717613196
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 62.182991610544796
      },
      "gaming": {
       "accuracy": 0.6896551724137931,
       "cc": 62.500443201241886,
       "cc_by_type": {
        "choice": 47.41035856573706,
        "noul": 77.77777777777777,
        "score": 62.31319326021083
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 62.500443201241886
      },
      "insurance_claims": {
       "accuracy": 0.6612903225806451,
       "cc": 43.90627215367682,
       "cc_by_type": {
        "choice": 37.639198218262806,
        "noul": 64.70588235294117,
        "score": 29.37373588982647
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 43.90627215367682
      },
      "knowledge_graphs": {
       "accuracy": 0.7761194029850746,
       "cc": 67.50803741109404,
       "cc_by_type": {
        "choice": 73.01349325337331,
        "noul": 84.61538461538463,
        "score": 44.895234364524185
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 67.50803741109404
      },
      "lead_generation": {
       "accuracy": 0.88,
       "cc": 81.29462146691996,
       "cc_by_type": {
        "choice": 78.84344146685473,
        "noul": 82.85714285714285,
        "score": 82.18328007676227
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 81.29462146691996
      },
      "legal_compliance": {
       "accuracy": 0.8214285714285714,
       "cc": 75.55224399955789,
       "cc_by_type": {
        "choice": 75.92778335005015,
        "noul": 85.71428571428572,
        "score": 65.01466293433779
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 75.55224399955789
      },
      "llm_guardrails": {
       "accuracy": 0.9436619718309859,
       "cc": 87.29831013388969,
       "cc_by_type": {
        "choice": 91.85336048879836,
        "noul": 100,
        "score": 70.04156991287071
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 87.29831013388969
      },
      "model_routing": {
       "accuracy": 0.8513513513513513,
       "cc": 77.14822730255678,
       "cc_by_type": {
        "choice": 85.97888177254633,
        "noul": 88.23529411764706,
        "score": 57.2305060174769
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 77.14822730255678
      },
      "moderation_trust_safety": {
       "accuracy": 0.8813559322033898,
       "cc": 80.46225649434656,
       "cc_by_type": {
        "choice": 73.8562091503268,
        "noul": 88.23529411764706,
        "score": 79.29526621506582
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 80.46225649434656
      },
      "other": {
       "accuracy": 0.78125,
       "cc": 74.83573143972326,
       "cc_by_type": {
        "choice": 60.04439511653718,
        "noul": 100,
        "score": 64.46279920263258
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 74.83573143972326
      },
      "recruiting": {
       "accuracy": 0.8615384615384616,
       "cc": 77.38908697581972,
       "cc_by_type": {
        "choice": 74.04470079307859,
        "noul": 100,
        "score": 58.122560134380585
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 77.38908697581972
      },
      "risk_assessment": {
       "accuracy": 0.75,
       "cc": 67.34860681221112,
       "cc_by_type": {
        "choice": 56.06694560669455,
        "noul": 84.61538461538463,
        "score": 61.36349021455414
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 67.34860681221112
      },
      "scientific_discovery": {
       "accuracy": 0.7142857142857143,
       "cc": 58.858165482863285,
       "cc_by_type": {
        "choice": 63.190184049079754,
        "noul": 60.00000000000001,
        "score": 53.384312399510094
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 58.858165482863285
      },
      "search_retrieval": {
       "accuracy": 0.9701492537313433,
       "cc": 88.42138573901913,
       "cc_by_type": {
        "choice": 100,
        "noul": 71.42857142857142,
        "score": 93.835585788486
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 88.42138573901913
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.42153836090922,
      93.29445815493874
     ],
     "I": [
      61.588608709782534,
      70.77179554485214
     ],
     "capability": [
      75.39757168561711,
      81.50475046575048
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 92.92233551579638,
     "capability": 79.8588198387464,
     "intelligence": 66.79530416169641
    },
    "per_type": {
     "choice": {
      "accuracy": 0.788,
      "calibration": 91.27931159872443,
      "cc": 71.08243829000413,
      "cc_by_type": {
       "choice": 71.08243829000413
      },
      "cc_tiered": 71.66634885305376,
      "cc_tiered_open": 71.16728353223091,
      "cc_tiered_sealed": 72.1654141738766,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 71.08243829000413
     },
     "noul": {
      "accuracy": 0.8933333333333333,
      "calibration": 94.09912457008461,
      "cc": 78.66666666666666,
      "cc_by_type": {
       "noul": 78.66666666666666
      },
      "cc_tiered": 72.4842564537357,
      "cc_tiered_open": 67.42028985507247,
      "cc_tiered_sealed": 77.54822305239892,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 78.66666666666666
     },
     "score": {
      "accuracy": 0.7573333333333333,
      "calibration": 93.38857037858008,
      "cc": 65.88786028381091,
      "cc_by_type": {
       "score": 65.88786028381091
      },
      "cc_tiered": 56.23530717829977,
      "cc_tiered_open": 50.77833268141265,
      "cc_tiered_sealed": 61.69228167518689,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 65.88786028381091
     }
    },
    "raw_on_A": {
     "calibration": 87.00897174892356,
     "capability": 76.82662300947624,
     "intelligence": 66.6442742700289
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 75.53207295041179,
    "Emphasis on Accuracy 60:20:20": 71.77673849105302,
    "Emphasis on Cost 20:20:60": 73.96722082675578,
    "Emphasis on Speed 20:60:20": 81.52195397957912,
    "Intelligence only": 66.79530416169641
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Arcbright (arcbright on Hugging Face)",
   "axes": {
    "calibration": 91.63064878963459,
    "cost": 55.0955403040988,
    "intelligence": 71.94618834920504,
    "speed": 89.86892425290594
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.045599768064632624,
      "ece_score": 90.88004638707348,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 90.88004638707348,
      "score_before_time_limit": 90.88004638707348,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.07020344233521425,
      "ece": 0.03816948771665763,
      "n": 373,
      "score": 92.36610245666847,
      "score_before_time_limit": 92.36610245666847,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9811827956989247,
      "interval90_width": 0.39986559139784944,
      "n": 372,
      "nrps": 0.06587467173838109,
      "score": 91.64579752516181,
      "score_before_time_limit": 91.64579752516181,
      "time_limit_failed_share": 0,
      "top_ece": 0.05060468887919142
     }
    },
    "score": 91.63064878963459
   },
   "capability": 81.78841856941982,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 777,809 input + 1,485 output tokens over 1,485 decisions = 524.777104377 tokens/decision. GPU-time cost USD 0.0313917352653123/1,000; old list-price cost USD 0.0474799393939394/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0313917352653123/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.047005139999999994
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.031391735265312345
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Arcbright Elf 1 31B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.525/h. The author's own `python3 -m elf_serve` from the pinned commit on 127.0.0.1:8000 inside a network-none Docker container with the cut proven from inside (loopback only, no default route, DNS and TCP fail), offline and credential-free, one request at a time. Timed over loopback on the VM. Measured 22 Oct 2026.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 74.68088774011248,
    "I_sealed": 69.2114889582976,
    "base": 71.94618834920504,
    "excess": -4.370127988540823,
    "gap": 5.469398781814888,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 76.10361879897965,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 75.46398238439761,
       "judge": 64.40342781806197,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 91,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 100,
       "judge": 70,
       "standard": 100
      }
     },
     "open|score": {
      "cc": 56.9390444213578,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.89638859502351,
       "hard": 64.43144230077161,
       "judge": 48.69808364892321,
       "standard": 48.83701773434921
      }
     },
     "sealed|choice": {
      "cc": 80.12686413307563,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 75.02023823291314,
       "judge": 79.72575905974536,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 77.86377492810843,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 70.9090909090909,
       "judge": 81.53846153846153,
       "standard": 82.08955223880596
      }
     },
     "sealed|score": {
      "cc": 49.643827813708725,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 73.05883560337949,
       "hard": 46.65352270756829,
       "judge": 48.28621943708376,
       "standard": 45.95334669609168
      }
     }
    }
   },
   "jevbench_score": 73.95046410857292,
   "key": "arcbright-elf-1-31b",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (model card, root LICENSE and serve/LICENSE; base google/gemma-4-31B-it under the Gemma 4 licence), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "arcbright/elf-1-31b@8c7799280c8c04a3802af1feeb3a1ad40d6c4eeb",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9115281501340483,
    "decisive": 373,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 33,
   "ranked": true,
   "ranks": {
    "A": 33,
    "B": 22,
    "C": 28,
    "capability": 22
   },
   "repo": "https://huggingface.co/arcbright/elf-1-31b",
   "scores": {
    "A": 73.95046410857292,
    "B": 73.540724842297,
    "C": 73.95046410857292
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27325655800009374,
    "p50_s_raw": 0.06162827900004686,
    "p95_s_adjusted": 0.37716986049996426,
    "p95_s_raw": 0.11358493024998212
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The author's own `elf_serve` package at the pinned revision 8c7799280c8c04a3802af1feeb3a1ad40d6c4eeb, unmodified: serve/elf_serve/{__init__,__main__,joint,render,server,settings,worker}.py, byte-identical (sha256, 7/7) to the arcbright/elf-1-12b serve files measured 20 Oct; the author's serve/SHA256SUMS verifies 11/11 inside the container. Started as `python3 -m elf_serve` with ELF_MODEL pointed at the commit-pinned local snapshot and the repo-root decision_config.json (the configuration of the author's Decision Index run: hybrid-v1, online FP8 from the bf16 checkpoint, max_model_len 32,768, temperatures scalar 2.186 / choice 1.7509 / noul 1.1758 / score 2.1257, noul_commit [0.199, 0.801]); ELF_CONFIG was never set, so serve/configs/elf-1-31b-prod.json was not used. Stock digest-pinned vllm/vllm-openai:v0.30.0 image plus the same hash-pinned wheels as the 12B row (offline --no-deps install). The two audited patterns (two read-only constant nvidia-smi queries in settings.py; a snapshot_download fallback unreachable as measured, network-none container) ride along unchanged: The benchmark owner cleared these exact bytes for this one measurement before the run. Weights: the commit-pinned snapshot, every file sha256-verified on the host and again in the container. FIRST MEASUREMENT (found by our own intake: Hugging Face decision-model tag / Decision Index). AUTHOR-DEFAULT CONTEXT CAP: the shipped decision_config.json caps a request row at 32,768 tokens; the author's server returns a real HTTP 422 above it, never a truncation. The 15 items of 76k-82k tokens were refused exactly as pre-registered and are scored wrong; the largest answered item is 1,828 tokens. ONLINE FP8: the author's Decision Index configuration quantises the bf16 checkpoint to FP8 at load time; this row is that configuration. NOUL COMMIT: the shipped config moves an interior noul probability to the nearer of 0.199 / 0.801 (author default, kept). SAME CODE AS ELF 1 12B: the seven serve files are byte-identical to the measured arcbright-elf-1-12b row. AUDITED CODE PATTERNS: see code_revision. TRAINING DATA, the author's own statement: one LoRA on the attention layers merged into bf16, 38,000 rows from public datasets incl. ZefanCai/Open-Jev and Open-Jev-v1.1 plus own generators, including the train splits of six Decision Index benchmarks (HoVer, WinoGrande, When2Call, GSM8K, BANKING77, CLINC150); the author states it screened for overlap with Decision Index test items - not independently verified here. SELF-REPORTED, unverified: 84 ms median latency (FP8, one RTX PRO 6000) on the author's card. 10 s RULE: max measured decision latency 0.24 s; no decision near the limit.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 57.204585699967765,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": -20.052909566763375
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 57.204585699967765
      },
      "calibration_gen": {
       "accuracy": 0.8118811881188119,
       "cc": 67.97203660818023,
       "cc_by_type": {
        "choice": 84.09331919406152,
        "noul": 60.00000000000001,
        "score": 59.822790630479176
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 67.97203660818023
      },
      "composite_gen": {
       "accuracy": 0.8712871287128713,
       "cc": 80.33590291986127,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 85.18518518518519,
        "score": 62.326588615049026
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 80.33590291986127
      },
      "fairness": {
       "accuracy": 1,
       "cc": 95.78198861246427,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 87.34596583739285
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 95.78198861246427
      },
      "jaggedness": {
       "accuracy": 0.6777777777777778,
       "cc": 55.1268918875868,
       "cc_by_type": {
        "choice": 56.689124157844084,
        "noul": 67.56756756756756,
        "score": 41.12398393734876
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 55.1268918875868
      },
      "llm_router": {
       "accuracy": 0.8945686900958466,
       "cc": 80.79446793487303,
       "cc_by_type": {
        "choice": 90.49044642070969,
        "noul": 95.8762886597938,
        "score": 56.01666872411562
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 80.79446793487303
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 54.17489074061117,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 75,
        "score": 24.63807428368917
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 54.17489074061117
      },
      "multilingual": {
       "accuracy": 0.7890909090909091,
       "cc": 69.06341467802686,
       "cc_by_type": {
        "choice": 76.5242908379524,
        "noul": 77.77777777777777,
        "score": 52.8881754183504
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.06341467802686
      },
      "negation_pair_gen": {
       "accuracy": 0.9069767441860465,
       "cc": 76.81877153861538,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 80,
        "score": 57.12298128251282
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 76.81877153861538
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 80.21167381519848,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 55.650036460610465
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.21167381519848
      },
      "usecase": {
       "accuracy": 0.7318181818181818,
       "cc": 65.92152623993582,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 75,
        "score": 49.72536303353292
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 65.92152623993582
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 24.201077194803236,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": -20.25391127273315
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 24.201077194803236
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 81.72251651210776,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 45.16754953632326
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 81.72251651210776
      },
      "de": {
       "accuracy": 0.875,
       "cc": 86.17845480227358,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 74.46456794664377
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 86.17845480227358
      },
      "el": {
       "accuracy": 0.8,
       "cc": 76.81907343233173,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 97.31010670295426
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 76.81907343233173
      },
      "en": {
       "accuracy": 0.8192875760208514,
       "cc": 71.79392556276699,
       "cc_by_type": {
        "choice": 80.66901795194102,
        "noul": 81.29496402877699,
        "score": 53.417794707582935
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 71.79392556276699
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 67.4304993100405,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 21.867354373840953
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.4304993100405
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 64.6764267214245,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 93.48283207684182
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 64.6764267214245
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 81.07498866843883,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 64.4014365935518
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 81.07498866843883
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 72.81153344833051,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100,
        "score": 18.4346003449915
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 72.81153344833051
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 59.19215964961905,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 5.635715114094886
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 59.19215964961905
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 50.387000946248946,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 80,
        "score": -0.24487484195212517
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 50.387000946248946
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 92.19353263244655,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 76.58059789733964
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 92.19353263244655
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.70832869724404,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 93.7916527583988
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.70832869724404
      },
      "nl": {
       "accuracy": 0.8,
       "cc": 71.38116254353474,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 100,
        "score": 40.117513604630204
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 71.38116254353474
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 74.49489472134356,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80,
        "score": 51.1021196274247
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 74.49489472134356
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.77618028751725,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 44.149401204331326
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.77618028751725
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 49.68215630779286,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 14.578383816995565
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 49.68215630779286
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 66.88714612791158,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100,
        "score": 44.13969925329998
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 66.88714612791158
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 77.14435815588773,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 63.69113898379224
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 77.14435815588773
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 76.32837381348317,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 28.98512144044949
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 76.32837381348317
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8453333333333334,
       "cc": 80.02257984270653,
       "cc_by_type": {
        "choice": 80.02257984270653
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 80.02257984270653
      },
      "noul": {
       "accuracy": 0.9066666666666666,
       "cc": 81.33333333333333,
       "cc_by_type": {
        "noul": 81.33333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 81.33333333333333
      },
      "score": {
       "accuracy": 0.656,
       "cc": 52.04184226648492,
       "cc_by_type": {
        "score": 52.04184226648492
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 52.04184226648492
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7727272727272727,
       "cc": 59.115790129793766,
       "cc_by_type": {
        "choice": 68,
        "noul": 75,
        "score": 34.34737038938128
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 59.115790129793766
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 78.39539955294666,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 35.18619865883996
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 78.39539955294666
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 50.139870119207636,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": -17.060064439125068
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 50.139870119207636
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 13.160050611862175,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": -0.28593003575849707
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 13.160050611862175
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 61.53698928966147,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 65.7703881588395
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 61.53698928966147
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 52.01161833869051,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 100,
        "score": 6.34932042487657
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 52.01161833869051
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 47.27061808024164,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 28.3664760894644
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 47.27061808024164
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 39.03343953882915,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0,
        "score": 48.459203633909055
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 39.03343953882915
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 59.09317237442428,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 100,
        "score": 39.77951712327283
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 59.09317237442428
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 69.13866057105047,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100,
        "score": 45.95987892942547
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 69.13866057105047
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 66.47674606607687,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 21.652460420452833
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 66.47674606607687
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 49.12803388587864,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100,
        "score": 44.15829520602301
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 49.12803388587864
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 68.9448826497521,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 58.35096272852691
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.9448826497521
      },
      "model_routing": {
       "accuracy": 0.880184331797235,
       "cc": 79.17182117069517,
       "cc_by_type": {
        "choice": 92.98701298701299,
        "noul": 100,
        "score": 44.528450525072486
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.17182117069517
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 52.43173568281341,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0,
        "score": 68.81152182771086
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 52.43173568281341
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 60.80044195437011,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 65.78108445178256
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 60.80044195437011
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 56.63512118362161,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 100,
        "score": -0.5379862028297433
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 56.63512118362161
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 50.0779780831841,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 45.88610816259578
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 50.0779780831841
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.615459389366,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.84637816809803
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.615459389366
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.37717693640397,
      92.31194864397001
     ],
     "I": [
      68.15049227545063,
      75.01758533219932
     ],
     "capability": [
      78.3609233982891,
      83.20686623472555
     ],
     "composite": null,
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.63064878963459,
     "capability": 81.78841856941982,
     "intelligence": 71.94618834920504
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8453333333333334,
      "calibration": 90.88004638707348,
      "cc": 80.02257984270653,
      "cc_by_type": {
       "choice": 80.02257984270653
      },
      "cc_tiered": 78.11524146602764,
      "cc_tiered_open": 76.10361879897965,
      "cc_tiered_sealed": 80.12686413307563,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 80.02257984270653
     },
     "noul": {
      "accuracy": 0.9066666666666666,
      "calibration": 92.36610245666847,
      "cc": 81.33333333333333,
      "cc_by_type": {
       "noul": 81.33333333333333
      },
      "cc_tiered": 84.43188746405421,
      "cc_tiered_open": 91,
      "cc_tiered_sealed": 77.86377492810843,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 81.33333333333333
     },
     "score": {
      "accuracy": 0.656,
      "calibration": 91.64579752516181,
      "cc": 52.04184226648492,
      "cc_by_type": {
       "score": 52.04184226648492
      },
      "cc_tiered": 53.291436117533266,
      "cc_tiered_open": 56.9390444213578,
      "cc_tiered_sealed": 49.643827813708725,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.04184226648492
     }
    },
    "raw_on_A": {
     "calibration": 89.21683012213103,
     "capability": 80.3975934709685,
     "intelligence": 71.57835681980598
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.48162472186283,
    "Emphasis on Accuracy 60:20:20": 70.44690662327388,
    "Emphasis on Cost 20:20:60": 62.91091678655649,
    "Emphasis on Speed 20:60:20": 76.41577367127479,
    "Intelligence only": 71.94618834920504
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "IBOU-SEARCH",
   "axes": {
    "calibration": 82.22627427476863,
    "cost": 75.23279561612854,
    "intelligence": 27.956374646954437,
    "speed": 89.52285293002008
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13190589669297953,
      "ece_score": 73.6188206614041,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 73.6188206614041,
      "score_before_time_limit": 73.6188206614041,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.581769436997319,
      "brier": 0.1619943265239912,
      "ece": 0.0674130307496136,
      "n": 373,
      "score": 86.51739385007728,
      "score_before_time_limit": 86.51739385007728,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9919354838709677,
      "interval90_width": 0.7842741935483872,
      "n": 372,
      "nrps": 0.1214577533167964,
      "score": 86.54260831282451,
      "score_before_time_limit": 86.54260831282451,
      "time_limit_failed_share": 0,
      "top_ece": 0.07384504021335675
     }
    },
    "score": 82.22627427476863
   },
   "capability": 55.09132446086153,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,011,369 input + 0 output tokens over 1,485 decisions = 681.056565657 tokens/decision. GPU-time cost USD 0.00669227038807155/1,000; old list-price cost USD 0.020431696969697/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00669227038807155/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.02022738
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.006692270388071554
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ArseneLupin v1.1 (IBOU-SEARCH, Qwen3.5-4B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's `python -m arsenelupin serve` (engine shared_prefix, bf16, calibration T 2.45) behind the wire-name proxy.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 33.63845928917361,
    "I_sealed": 22.274290004735263,
    "base": 27.956374646954437,
    "excess": 1.524642514082636,
    "gap": 11.364169284438347,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 66.71937273976232,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 54.70273670965713,
       "judge": 72.31377719182598,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 4.347826086956519,
       "judge": -40,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 34.196005127758525,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.46334077824246,
       "hard": 22.992987523752227,
       "judge": 43.70520186247461,
       "standard": 41.20457740845502
      }
     },
     "sealed|choice": {
      "cc": 47.96190215319236,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 56.16264294790343,
       "hard": 44.836359431016525,
       "judge": 48.87365328109698,
       "standard": 48.74499050833158
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 10.344827586206895,
       "hard": -30.909090909090907,
       "judge": -53.84615384615385,
       "standard": -22.388059701492537
      }
     },
     "sealed|score": {
      "cc": 18.860967861013435,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 32.91947880531089,
       "hard": 16.11073859508191,
       "judge": 16.587666801933665,
       "standard": 20.742122509347404
      }
     }
    }
   },
   "jevbench_score": 17.27345398717365,
   "key": "arsenelupin-v1.1",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "IBOU-SEARCH/ArseneLupin-v1.1@411a6aefab87a4c155372c2a0a20777c2f5e1317",
   "noul_decisive": {
    "acc_among_decisive": 0.9423076923076923,
    "decisive": 156,
    "decisive_rate": 0.41823056300268097,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.22627427476863,
     "cost": 60.69086683660791,
     "intelligence": 27.956374646954437,
     "speed": 89.69506927060962
    },
    "board_release": "v1.6.43",
    "capability": 55.09132446086153,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 82,
    "capability_rank": 108,
    "composite_ci95": {
     "A": [
      12.971040788669306,
      20.8144338578416
     ],
     "B": [
      10.817216180981147,
      17.96790173408304
     ],
     "C": [
      9.007667214353683,
      14.454467956834444
     ]
    },
    "jevbench_score": 16.550252807567574,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": 114,
    "rank": 151,
    "ranks": {
     "A": 151,
     "B": 152,
     "C": 151,
     "capability": 163
    },
    "scores": {
     "A": 16.550252807567574,
     "B": 14.040724977478481,
     "C": 11.493231116366374
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3189419380272739,
     "p50_s_raw": 0.08447096901363693,
     "p95_s_adjusted": 0.3363422646856634,
     "p95_s_raw": 0.0931711323428317
    },
    "views": {
     "Balanced 33:33:33": 14.793881978653419,
     "Emphasis on Accuracy 60:20:20": 11.584152166235047,
     "Emphasis on Cost 20:20:60": 16.223374091593428,
     "Emphasis on Speed 20:60:20": 18.240772757738373,
     "Intelligence only": 8.739821181204116
    }
   },
   "rank": 194,
   "ranked": true,
   "ranks": {
    "A": 194,
    "B": 197,
    "C": 195,
    "capability": 197
   },
   "repo": "https://huggingface.co/IBOU-SEARCH/ArseneLupin-v1.1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 17.27345398717365,
    "B": 14.45136735621717,
    "C": 11.995454157759482
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3199281820037868,
    "p50_s_raw": 0.0849640910018934,
    "p95_s_adjusted": 0.34886886222520835,
    "p95_s_raw": 0.09943443111260418
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 50.51872953772545,
       "cc_by_type": {
        "choice": 50,
        "noul": 100,
        "score": 1.5561886131763458
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 50.51872953772545
      },
      "calibration_gen": {
       "accuracy": 0.38613861386138615,
       "cc": -1.2802984282284164,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": -28.000000000000004,
        "score": 3.692508745007217
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.504950495049505,
       "cc": 9.769891325776209,
       "cc_by_type": {
        "choice": 57.72357723577236,
        "noul": -62.96296296296296,
        "score": 34.54905970451922
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 9.769891325776209
      },
      "fairness": {
       "accuracy": 0.6875,
       "cc": 54.72513077918578,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 44.17539233755736
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 54.72513077918578
      },
      "jaggedness": {
       "accuracy": 0.3055555555555556,
       "cc": -19.677183728618825,
       "cc_by_type": {
        "choice": 13.378248315688179,
        "noul": -67.56756756756756,
        "score": -4.842231933977081
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.6932907348242812,
       "cc": 46.00272325098312,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 34.020618556701024,
        "score": 34.89360032894183
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 46.00272325098312
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -12.636857704693645,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -75,
        "score": 23.687365030248962
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5563636363636364,
       "cc": 19.72366546133326,
       "cc_by_type": {
        "choice": 56.9611998695794,
        "noul": -36.111111111111114,
        "score": 38.3209076255315
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 19.72366546133326
      },
      "negation_pair_gen": {
       "accuracy": 0.36046511627906974,
       "cc": -11.788936369121076,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -73.33333333333334,
        "score": 4.633190892636774
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.796875,
       "cc": 63.02665837969059,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 60.00000000000001,
        "score": 53.10399916309577
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 63.02665837969059
      },
      "usecase": {
       "accuracy": 0.45,
       "cc": 8.571271059852021,
       "cc_by_type": {
        "choice": 50.98039215686277,
        "noul": -45.833333333333336,
        "score": 20.566754356026628
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 8.571271059852021
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": 13.512897780745016,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": 50,
        "score": 4.824407627949334
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 13.512897780745016
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 4.233223448197042,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 12.69967034459113
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 4.233223448197042
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 39.83897361366874,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 0,
        "score": 43.41072615074075
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 39.83897361366874
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 4.953090501411556,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -100,
        "score": 81.89837764948606
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 4.953090501411556
      },
      "en": {
       "accuracy": 0.5178105994787141,
       "cc": 15.967276196171342,
       "cc_by_type": {
        "choice": 50.3545233765758,
        "noul": -20.863309352517987,
        "score": 18.410614564456218
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 15.967276196171342
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 61.117636296678036,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75,
        "score": 18.140837111893816
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 61.117636296678036
      },
      "fi": {
       "accuracy": 0.25,
       "cc": 2.2535777645684334,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 38.45472236474355
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 2.2535777645684334
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 29.96031603152154,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0,
        "score": 53.410359859270486
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 29.96031603152154
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 20.903606766234788,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -75,
        "score": 54.470596835017204
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 20.903606766234788
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 43.96472119872376,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 33.33333333333333,
        "score": 12.590448345456796
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 43.96472119872376
      },
      "ja": {
       "accuracy": 0.44,
       "cc": 7.47713527194653,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": 0,
        "score": 8.21377277374269
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 7.47713527194653
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -10.20949871018135,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 13.815948313900407
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 93.5496427429219,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 80.64892822876573
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 93.5496427429219
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -13.055905252050016,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -100,
        "score": 38.75436216592787
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5641025641025641,
       "cc": 19.62379626604363,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -40,
        "score": 29.34113065170686
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 19.62379626604363
      },
      "pt": {
       "accuracy": 0.5714285714285714,
       "cc": 22.01660632547086,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": -19.999999999999996,
        "score": 39.08458621271193
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 22.01660632547086
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": -7.000724261111401,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60,
        "score": 41.12548679113388
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": 3.4888991168227768,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -50,
        "score": 25.68408865481615
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 3.4888991168227768
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 83.92575718462467,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 51.777271553874016
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 83.92575718462467
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 9.15759259136832,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -13.77016007900238
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.15759259136832
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6253333333333333,
       "cc": 51.60642186034941,
       "cc_by_type": {
        "choice": 51.60642186034941
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 51.60642186034941
      },
      "noul": {
       "accuracy": 0.392,
       "cc": -21.599999999999998,
       "cc_by_type": {
        "noul": -21.599999999999998
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.4693333333333333,
       "cc": 22.200266391018797,
       "cc_by_type": {
        "score": 22.200266391018797
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 22.200266391018797
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -15.830990825213652,
       "cc_by_type": {
        "choice": 52,
        "noul": -75,
        "score": -24.49297247564095
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": 1.7168943804986962,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -33.333333333333336,
        "score": 13.018177965512656
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 1.7168943804986962
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 28.7536967232783,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": 18.78141537308693
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 28.7536967232783
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -25.640628784241386,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": 28.34127154201269
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 11.906917556526437,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 27.025100495666265
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 11.906917556526437
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -28.821393224240854,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -100,
        "score": -10.99248155951502
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 20.561134748485653,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -42.85714285714286,
        "score": 33.95231180848217
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 20.561134748485653
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -17.75792051210721,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100,
        "score": 19.896970170995452
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": 5.325356796383858,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 38.47607038915157
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 5.325356796383858
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -12.492206719922391,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -33.333333333333336,
        "score": -1.3595609163695954
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 20.266776521258546,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": 5.761569873853178
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 20.266776521258546
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -20.09225148226946,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": 4.239374585449685
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 5.74928641158261,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -60,
        "score": 23.313118351830365
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 5.74928641158261
      },
      "model_routing": {
       "accuracy": 0.7004608294930875,
       "cc": 45.029199454317904,
       "cc_by_type": {
        "choice": 76.62337662337664,
        "noul": 39.393939393939405,
        "score": 19.07028234563767
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 45.029199454317904
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 1.9206201017423343,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 51.82711942230953
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 1.9206201017423343
      },
      "recruiting": {
       "accuracy": 0.5555555555555556,
       "cc": 20.15666397470658,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 27.229509101464224
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 20.15666397470658
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 11.531424940662966,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50,
        "score": 14.150925068294306
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 11.531424940662966
      },
      "scientific_discovery": {
       "accuracy": 0.30434782608695654,
       "cc": -17.435804548794813,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -100,
        "score": 20.15635446955759
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.6666666666666666,
       "cc": 15.043192447509194,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": -100,
        "score": 66.48189050978736
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 15.043192447509194
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.84964659068773,
      84.32731575272373
     ],
     "I": [
      25.32894837642683,
      30.711526629669294
     ],
     "capability": [
      52.11423594286504,
      56.58071014414595
     ],
     "composite": null,
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.22627427476863,
     "capability": 55.09132446086153,
     "intelligence": 27.956374646954437
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6253333333333333,
      "calibration": 73.6188206614041,
      "cc": 51.60642186034941,
      "cc_by_type": {
       "choice": 51.60642186034941
      },
      "cc_tiered": 57.340637446477345,
      "cc_tiered_open": 66.71937273976232,
      "cc_tiered_sealed": 47.96190215319236,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 51.60642186034941
     },
     "noul": {
      "accuracy": 0.392,
      "calibration": 86.51739385007728,
      "cc": -21.599999999999998,
      "cc_by_type": {
       "noul": -21.599999999999998
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4693333333333333,
      "calibration": 86.54260831282451,
      "cc": 22.200266391018797,
      "cc_by_type": {
       "score": 22.200266391018797
      },
      "cc_tiered": 26.52848649438598,
      "cc_tiered_open": 34.196005127758525,
      "cc_tiered_sealed": 18.860967861013435,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 22.200266391018797
     }
    },
    "raw_on_A": {
     "calibration": 75.10885434845505,
     "capability": 52.09577580877465,
     "intelligence": 29.082697269094247
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.570860110345219,
    "Emphasis on Accuracy 60:20:20": 11.862249353250903,
    "Emphasis on Cost 20:20:60": 18.00483090690624,
    "Emphasis on Speed 20:60:20": 18.930111317149827,
    "Intelligence only": 8.739821181204116
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "attest",
   "axes": {
    "calibration": 73.84793106824938,
    "cost": 50.72941039177032,
    "intelligence": 65.23849472031306,
    "speed": 96.47488998719595
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08913950574898584,
      "ece_score": 82.17209885020283,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 82.17209885020283,
      "score_before_time_limit": 82.17209885020283,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.02680965147453085,
      "brier": 0.12374104570386348,
      "ece": 0.1233374127983535,
      "n": 373,
      "score": 75.3325174403293,
      "score_before_time_limit": 75.3325174403293,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.7016129032258065,
      "interval90_width": 0.0378584229390681,
      "n": 372,
      "nrps": 0.10002561109774619,
      "score": 64.03917691421601,
      "score_before_time_limit": 64.03917691421601,
      "time_limit_failed_share": 0,
      "top_ece": 0.3095954253089667
     }
    },
    "score": 73.84793106824938
   },
   "capability": 69.54321289428123,
   "composite_ci95": {
    "A": [
     65.6462584019398,
     69.5084228209243
    ],
    "B": [
     64.41799110254772,
     69.60746615454364
    ],
    "C": [
     64.92305184156147,
     69.5084228209243
    ]
   },
   "cost": {
    "basis": "ESTIMATE, v1.5-M2 base-model reference rule: vendor-hosted API, customer list price USD 0.048/M input and USD 0.00/M output effective 2026-10-11 has been in force fewer than 30 days. Base model google/gemma-4-31b-it reference on OpenRouter USD 0.09/M input / USD 0.34/M output, snapshot fetched 2026-10-12T17:55:05Z (SHA-256 81eae2f3f94bf299c8d4499260f842c357b7a869ce82dfa1741ad6a89cf57014), same base-model estimate precedent as metask_jev_rain_31b_trained. Applied to measured response usage; failed HTTP 500 usage/billing may be unreported. No output generation; reported output tokens 0. Estimated, not a vendor bill. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.043889 per 1,000 decisions (0.679x the official cost cap USD 0.064595), against USD 0.043450 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.043449839999999997
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04388872727272727
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "attest-31B-v1.0",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "paid evaluation. Vendor-hosted attest.engineering /v1/systemone API; latency measured from Sandy (Germany), using the official per-item latency subset without API adjustment. Full S1200+P300 draw, not API A300 supplement; no API equating. 15 of 1,500 requests returned HTTP 500 after 8 retry invocations and are scored as wrong in the full denominator. Public model documentation declares frozen google/gemma-4-31B-it @842da3794eaa0b77d5f08bae87a17459d91ff475, unmodified bf16, no adapter or quantization; single forward pass reading option-label logits, no generated tokens, one fitted temperature (value not documented). Hosting implementation and deployed weights were not independently inspected. Hosted API access is not an open-weights release by the vendor; base weights are under the Gemma 4 licence. ESTIMATE, v1.5-M2 base-model reference rule: vendor-hosted API, customer list price USD 0.048/M input and USD 0.00/M output effective 2026-10-11 has been in force fewer than 30 days. Base model google/gemma-4-31b-it reference on OpenRouter USD 0.09/M input / USD 0.34/M output, snapshot fetched 2026-10-12T17:55:05Z (SHA-256 81eae2f3f94bf299c8d4499260f842c357b7a869ce82dfa1741ad6a89cf57014), same base-model estimate precedent as metask_jev_rain_31b_trained. Applied to measured response usage; failed HTTP 500 usage/billing may be unreported. No output generation; reported output tokens 0. Estimated, not a vendor bill.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 64.92424666524875,
    "I_sealed": 65.55274277537737,
    "base": 65.23849472031306,
    "excess": -10.468022880484334,
    "gap": -0.6284961101286228,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 73.17162102453656,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 72.31377719182598,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 68.67632850241546,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 60.00000000000001,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 52.92479046879421,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.941722778327325,
       "hard": 60.749677220164536,
       "judge": 51.85551193839042,
       "standard": 33.370468606892665
      }
     },
     "sealed|choice": {
      "cc": 74.62818230752828,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 80.94027954256671,
       "hard": 71.89776801202729,
       "judge": 74.43682664054849,
       "standard": 77.2199957814807
      }
     },
     "sealed|noul": {
      "cc": 68.21259029184918,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 58.18181818181818,
       "judge": 72.30769230769232,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 53.817455726754645,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 78.07383467048999,
       "hard": 52.90470314143051,
       "judge": 47.46936317941805,
       "standard": 53.03691024654009
      }
     }
    }
   },
   "jevbench_score": 67.85694940132115,
   "key": "attest-31b-v1.0",
   "last_measured_on": "2026-10-12",
   "licence": "Hosted API; declared base weights under the Gemma 4 licence (no vendor weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.13",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475",
   "noul_decisive": {
    "acc_among_decisive": 0.8787878787878788,
    "decisive": 363,
    "decisive_rate": 0.9731903485254692,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 20,
    "axes": {
     "calibration": 73.84793106824938,
     "cost": 50.72941039177032,
     "intelligence": 65.23849472031306,
     "speed": 97.10993413996468
    },
    "board_release": "v1.6.43",
    "capability": 69.54321289428123,
    "capability_api_board_rank": 20,
    "capability_open_board_rank": null,
    "capability_rank": 49,
    "composite_ci95": {
     "A": [
      65.7193671470225,
      69.5903923988902
     ],
     "B": [
      64.47429646451545,
      69.67321341053939
     ],
     "C": [
      64.99583707590963,
      69.5903923988902
     ]
    },
    "jevbench_score": 67.9350679784662,
    "measured_in": "v1.6.13",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 38,
    "ranks": {
     "A": 38,
     "B": 33,
     "C": 32,
     "capability": 82
    },
    "scores": {
     "A": 67.9350679784662,
     "B": 67.37806588505295,
     "C": 67.9350679784662
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.10787388649987406,
     "p50_s_raw": 0.10787388649987406,
     "p95_s_adjusted": 0.18033925039962923,
     "p95_s_raw": 0.18033925039962923
    },
    "views": {
     "Balanced 33:33:33": 66.16905786253606,
     "Emphasis on Accuracy 60:20:20": 65.79366502170826,
     "Emphasis on Cost 20:20:60": 58.987811289065355,
     "Emphasis on Speed 20:60:20": 75.8338379652306,
     "Intelligence only": 65.23849472031306
    }
   },
   "rank": 95,
   "ranked": true,
   "ranks": {
    "A": 95,
    "B": 75,
    "C": 68,
    "capability": 104
   },
   "repo": "https://attest.engineering/model",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.85694940132115,
    "B": 67.31657701246269,
    "C": 67.85694940132115
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.11031594549967849,
    "p50_s_raw": 0.11031594549967849,
    "p95_s_adjusted": 0.20411396054878422,
    "p95_s_raw": 0.20411396054878422
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 91.66666666649981,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 99.99999999949942
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 91.66666666649981
      },
      "calibration_gen": {
       "accuracy": 0.7227722772277227,
       "cc": 65.68205659085977,
       "cc_by_type": {
        "choice": 61.82396606574761,
        "noul": 76,
        "score": 59.22220370683169
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 65.68205659085977
      },
      "composite_gen": {
       "accuracy": 0.8217821782178217,
       "cc": 71.7068789691002,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 62.962962962962955,
        "score": 58.66173898498804
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 71.7068789691002
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 90.47619047605701,
       "cc_by_type": {
        "choice": 71.42857142857143,
        "noul": 100,
        "score": 99.99999999959958
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 90.47619047605701
      },
      "jaggedness": {
       "accuracy": 0.5722222222222222,
       "cc": 41.84478952801769,
       "cc_by_type": {
        "choice": 45.13955726660251,
        "noul": 56.75675675675676,
        "score": 23.63805456069382
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 41.84478952801769
      },
      "llm_router": {
       "accuracy": 0.8977635782747604,
       "cc": 83.50801061677168,
       "cc_by_type": {
        "choice": 88.90552082416131,
        "noul": 93.81443298969072,
        "score": 67.804078036463
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.50801061677168
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 60.78261745433469,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 75,
        "score": 56.83238844547831
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 60.78261745433469
      },
      "multilingual": {
       "accuracy": 0.7563636363636363,
       "cc": 63.88819552264045,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": 61.111111111111114,
        "score": 58.91995736095108
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 63.88819552264045
      },
      "negation_pair_gen": {
       "accuracy": 0.7441860465116279,
       "cc": 41.72809466728673,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 6.666666666666665,
        "score": 25.184284001860192
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 41.72809466728673
      },
      "option_order": {
       "accuracy": 0.8958333333333334,
       "cc": 86.68442077230361,
       "cc_by_type": {
        "choice": 86.68442077230361
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 86.68442077230361
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 85.01995309361071,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 100,
        "score": 73.07787729885015
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.01995309361071
      },
      "usecase": {
       "accuracy": 0.7772727272727272,
       "cc": 69.59874631177811,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 75,
        "score": 57.080552660824544
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 69.59874631177811
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 25.657000371915586,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": -15.886141741396088
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 25.657000371915586
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 1,
       "cc": 99.83499124305172,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.50497372915514
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 99.83499124305172
      },
      "de": {
       "accuracy": 0.875,
       "cc": 88.7927420089141,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 90.27203133647683
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 88.7927420089141
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.60459342006904,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.9999999994995
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.60459342006904
      },
      "en": {
       "accuracy": 0.7793223284100782,
       "cc": 66.85046932152197,
       "cc_by_type": {
        "choice": 75.39693193883403,
        "noul": 70.50359712230217,
        "score": 54.650878903429714
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 66.85046932152197
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 66.69113187317208,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 19.649252063235643
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.69113187317208
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.84881602901042,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 99.99999999959955
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.84881602901042
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 75.35603715160619,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 68.42105263128916
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 75.35603715160619
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 81.9134078211331,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100,
        "score": 62.499999999712166
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 81.9134078211331
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 53.22313405090798,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -26.300979764657196
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 53.22313405090798
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 45.07973339782123,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 60.00000000000001,
        "score": 3.833322512764714
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 45.07973339782123
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 46.296506956290735,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 83.33396531331667
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 46.296506956290735
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.77777777761094,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.9999999994995
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.77777777761094
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 43.862463409737074,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 50.202108843929864
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 43.862463409737074
      },
      "no": {
       "accuracy": 0.5,
       "cc": 10.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 10.483870967741936
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 58.11281326204074,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 49.5733107129102
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 58.11281326204074
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 73.9630456140903,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 64.3169506313104
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 73.9630456140903
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 52.97872340417189,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 49.99999999974971
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 52.97872340417189
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 88.24218637816098,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 86.46568956926559
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 88.24218637816098
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.53664033508191,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 49.86798552137475
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.53664033508191
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 66.66666666666664,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -4.440892098500626e-14
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 66.66666666666664
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.804,
       "cc": 74.68378652480914,
       "cc_by_type": {
        "choice": 74.68378652480914
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 74.68378652480914
      },
      "noul": {
       "accuracy": 0.8506666666666667,
       "cc": 70.13333333333334,
       "cc_by_type": {
        "noul": 70.13333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 70.13333333333334
      },
      "score": {
       "accuracy": 0.6506666666666666,
       "cc": 55.14020270807265,
       "cc_by_type": {
        "score": 55.14020270807265
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 55.14020270807265
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 41.05013190841991,
       "cc_by_type": {
        "choice": 84,
        "noul": 0,
        "score": 39.15039572525971
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 41.05013190841991
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 69.18221735790132,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 7.546652073703974
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 69.18221735790132
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 33.73983739844072,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100,
        "score": -49.999999999799805
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 33.73983739844072
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 40.20929030056567,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": 100,
        "score": 25.891028796433858
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 40.20929030056567
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 59.627914855635204,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 74.53591847994909
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.627914855635204
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 81.67133102077554,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 100,
        "score": 95.32845847113167
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 81.67133102077554
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 51.05695837350043,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 39.72549696924077
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 51.05695837350043
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 31.06744966418366,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": -50,
        "score": 85.01419568244644
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 31.06744966418366
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 45.83333333332221,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 100,
        "score": -3.3373304120232206e-11
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 45.83333333332221
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 79.31347738618848,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100,
        "score": 63.636363636081526
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 79.31347738618848
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 72.4290843339358,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 51.13738220077384
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 72.4290843339358
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 74.35692248903656,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 87.58689649936777
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 74.35692248903656
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 78.81343307279933,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 76.44029921839794
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 78.81343307279933
      },
      "model_routing": {
       "accuracy": 0.8894009216589862,
       "cc": 82.42648273169006,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 100,
        "score": 56.63009754571953
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 82.42648273169006
      },
      "moderation_trust_safety": {
       "accuracy": 0.9444444444444444,
       "cc": 92.05979877042455,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 76.17939631127363
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 92.05979877042455
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 57.779434740179134,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 73.43115473121522
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.779434740179134
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 47.99092050610208,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50,
        "score": 23.52941176461164
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 47.99092050610208
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 59.19668873382481,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 73.24224011451793
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 59.19668873382481
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.99999999986125,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.99999999958375
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.99999999986125
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      69.51167624462902,
      77.41120372530025
     ],
     "I": [
      59.64991031101489,
      70.14236746435076
     ],
     "capability": [
      65.30829911391156,
      72.94151936366131
     ],
     "composite": {
      "A": [
       65.6462584019398,
       69.5084228209243
      ],
      "B": [
       64.41799110254772,
       69.60746615454364
      ],
      "C": [
       64.92305184156147,
       69.5084228209243
      ]
     },
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.84793106824938,
     "capability": 69.54321289428123,
     "intelligence": 65.23849472031306
    },
    "per_type": {
     "choice": {
      "accuracy": 0.804,
      "calibration": 82.17209885020283,
      "cc": 74.68378652480914,
      "cc_by_type": {
       "choice": 74.68378652480914
      },
      "cc_tiered": 73.89990166603242,
      "cc_tiered_open": 73.17162102453656,
      "cc_tiered_sealed": 74.62818230752828,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 74.68378652480914
     },
     "noul": {
      "accuracy": 0.8506666666666667,
      "calibration": 75.3325174403293,
      "cc": 70.13333333333334,
      "cc_by_type": {
       "noul": 70.13333333333334
      },
      "cc_tiered": 68.44445939713232,
      "cc_tiered_open": 68.67632850241546,
      "cc_tiered_sealed": 68.21259029184918,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 70.13333333333334
     },
     "score": {
      "accuracy": 0.6506666666666666,
      "calibration": 64.03917691421601,
      "cc": 55.14020270807265,
      "cc_by_type": {
       "score": 55.14020270807265
      },
      "cc_tiered": 53.37112309777443,
      "cc_tiered_open": 52.92479046879421,
      "cc_tiered_sealed": 53.817455726754645,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 55.14020270807265
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.07027859433674,
    "Emphasis on Accuracy 60:20:20": 65.7350327220277,
    "Emphasis on Cost 20:20:60": 58.94067738965652,
    "Emphasis on Speed 20:60:20": 75.60067159595273,
    "Intelligence only": 65.23849472031306
   }
  },
  {
   "adapter_id": "images/autojev-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 87.61544276592178,
    "cost": 56.735701109968524,
    "intelligence": 55.62422706919757,
    "speed": 88.30997360530876
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0794047531024999,
      "ece_score": 84.11904937950003,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 84.11904937950003,
      "score_before_time_limit": 84.11904937950003,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2964959568733153,
      "brier": 0.06886464230605442,
      "ece": 0.04566629583733653,
      "n": 371,
      "score": 90.8667408325327,
      "score_before_time_limit": 90.8667408325327,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 1,
      "interval90_width": 0.41666666666666674,
      "n": 369,
      "nrps": 0.0623820892760947,
      "score": 87.86053808573263,
      "score_before_time_limit": 87.86053808573263,
      "time_limit_failed_share": 0,
      "top_ece": 0.09020357450462646
     }
    },
    "score": 87.61544276592178
   },
   "capability": 71.61983491755967,
   "class": "unclassified",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 874,592 input + 0 output tokens over 1,624 decisions = 538.541871921 tokens/decision. GPU-time cost USD 0.0276785475398351/1,000; old list-price cost USD 0.226187586206897/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0276785475398351/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02767854753983507
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "AutoJev-27B (denis-pplx, Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 56.79842954079213,
    "I_sealed": 54.45002459760302,
    "base": 55.62422706919757,
    "excess": -0.2249826992491344,
    "gap": 2.3484049431891094,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 79.34292176773366,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 75.46398238439761,
       "judge": 80.22412656558997,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 34.36853002070393,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 56.68383683393881,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 65.12454762185703,
       "hard": 56.31402454129169,
       "judge": 64.63259495424096,
       "standard": 41.2799688448207
      }
     },
     "sealed|choice": {
      "cc": 75.98476401702004,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 68.43891402714931,
       "judge": 78.58472998137803,
       "standard": 84.48084739499939
      }
     },
     "sealed|noul": {
      "cc": 33.77589728457613,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": 12.621359223300965,
       "judge": 42.10526315789473,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 53.589412491212904,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.78624407613681,
       "hard": 41.90521556676637,
       "judge": 62.130233048417736,
       "standard": 56.54815971183675
      }
     }
    }
   },
   "jevbench_score": 68.56285633198503,
   "key": "autojev-27b",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9885057471264368,
    "decisive_rate": 0.7035040431266847,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.61544276592178,
     "cost": 29.365937020749882,
     "intelligence": 55.62422706919757,
     "speed": 88.60757446254297
    },
    "board_release": "v1.6.43",
    "capability": 71.61983491755967,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      17.95132364974654,
      18.715305818590913
     ],
     "B": [
      17.85214158412037,
      19.07667324580802
     ],
     "C": [
      12.749857516255931,
      18.58395156303398
     ]
    },
    "jevbench_score": 18.463427917707932,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 105,
    "rank": 140,
    "ranks": {
     "A": 140,
     "B": 135,
     "C": 133,
     "capability": 65
    },
    "scores": {
     "A": 18.463427917707932,
     "B": 18.603781697586662,
     "C": 15.868570767928437
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3447543190326542,
     "p50_s_raw": 0.09737715951632708,
     "p95_s_adjusted": 0.3996987941907718,
     "p95_s_raw": 0.12484939709538592
    },
    "views": {
     "Balanced 33:33:33": 16.343749600153146,
     "Emphasis on Accuracy 60:20:20": 17.37362601712716,
     "Emphasis on Cost 20:20:60": 13.123431991427351,
     "Emphasis on Speed 20:60:20": 20.08099797467756,
     "Intelligence only": 19.187204603319074
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 91,
   "ranked": true,
   "ranks": {
    "A": 91,
    "B": 91,
    "C": 88,
    "capability": 82
   },
   "repo": "https://huggingface.co/denis-pplx/autojev-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.56285633198503,
    "B": 65.51499645183391,
    "C": 58.9270065452981
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3509636391885579,
    "p50_s_raw": 0.10048181959427893,
    "p95_s_adjusted": 0.42047532487194983,
    "p95_s_raw": 0.13523766243597493
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.93064258139867,
      88.05572734371755
     ],
     "I": [
      50.54754866954701,
      59.786942555692185
     ],
     "capability": [
      67.70288142885737,
      72.92248557654668
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.61544276592178,
     "capability": 71.61983491755967,
     "intelligence": 55.62422706919757
    },
    "per_type": {
     "choice": {
      "accuracy": 0.82,
      "calibration": 84.11904937950003,
      "cc": 76.98057084395099,
      "cc_by_type": {
       "choice": 76.98057084395099
      },
      "cc_tiered": 77.66384289237685,
      "cc_tiered_open": 79.34292176773366,
      "cc_tiered_sealed": 75.98476401702004,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.98057084395099
     },
     "noul": {
      "accuracy": 0.688,
      "calibration": 90.8667408325327,
      "cc": 37.59999999999999,
      "cc_by_type": {
       "noul": 37.59999999999999
      },
      "cc_tiered": 34.07221365264003,
      "cc_tiered_open": 34.36853002070393,
      "cc_tiered_sealed": 33.77589728457613,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 37.59999999999999
     },
     "score": {
      "accuracy": 0.6506666666666666,
      "calibration": 87.86053808573263,
      "cc": 53.92178222051296,
      "cc_by_type": {
       "score": 53.92178222051296
      },
      "cc_tiered": 55.136624662575855,
      "cc_tiered_open": 56.68383683393881,
      "cc_tiered_sealed": 53.589412491212904,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.92178222051296
     }
    },
    "raw_on_A": {
     "calibration": 85.84423247942522,
     "capability": 73.42644032760776,
     "intelligence": 61.00864817579031
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.9289266566402,
    "Emphasis on Accuracy 60:20:20": 60.32624238306047,
    "Emphasis on Cost 20:20:60": 60.843322809903285,
    "Emphasis on Speed 20:60:20": 71.86529466426978,
    "Intelligence only": 55.62422706919758
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Remek Kinas (Remigiusz Kinas; Hugging Face Remek, GitHub rkinas)",
   "axes": {
    "calibration": 90.20661225231586,
    "cost": 55.924617993700764,
    "intelligence": 74.05810874808022,
    "speed": 86.63193265367512
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.042664316931708864,
      "ece_score": 91.46713661365823,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 90.24757479214279,
      "score_before_time_limit": 91.46713661365823,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.07774798927613946,
      "brier": 0.06996825861082419,
      "ece": 0.052891090232176925,
      "n": 373,
      "score": 88.94486578314562,
      "score_before_time_limit": 89.42178195356462,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9596774193548387,
      "interval90_width": 0.196236559139785,
      "n": 372,
      "nrps": 0.04515355626268863,
      "score": 91.42739618165918,
      "score_before_time_limit": 92.16471389280159,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.05577608294063997
     }
    },
    "score": 90.20661225231586
   },
   "capability": 82.13236050019805,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10285.272649 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=native). The row's own scored cost basis: 744,183 input + 0 output tokens over 1,485 decisions = 501.133333333 tokens/decision. GPU-time cost USD 0.0294563817924917/1,000; old list-price cost USD 0.045102/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0294563817924917/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.04465098000000001
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.029456381792491746
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "basal-omni-1.0-ultra (Remek Kinas, Gemma-4-31B-it + merged LoRA, typed-decision readout, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-17",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod, 1x NVIDIA H200 141 GB (sm_90, driver 580.173.02), Finland (EU), SPOT tier, USD 3.99/h, 17 Oct 2026; sealed run 08:59:30-09:25:01Z, one request at a time, no resumes. Speed is as measured on that card; the card's own figure is 80 ms p50 on an H200 (measured here: raw p50 0.107 s / p95 0.216 s over all 1,485 answered items; the Speed axis uses the official latency subset given in the row note, with the standing x2 + 0.15 s self-hosted adjustment on top). DEVICE DEVIATION: the pre-registered device was one non-preemptible 80 GB H100; no single-GPU H100 was rentable in the EU, so the run used this one H200 on the spot tier under a pre-registration amendment written before the sealed run (the card says about 32,000 tokens fit on a 141 GB H200 and about 10,000 on an 80 GB H100; the same 15 items were predicted to be refused on either card). Speed is therefore not strictly comparable to the H100 / RTX 5090 rows. Isolation: the author's server ran inside a network namespace with loopback only, with the cut PROVEN before any sealed item was dispatched (no interfaces but lo, no routes, outbound by name and by IP both failing) and proven again immediately before the first sealed request; credential-free environment (env -i, checked), HF offline flags, weights from a locally verified offline snapshot of the pinned revision (13 files, 62,578,671,091 bytes, every sha256 equal to the author's SHA256SUMS), sealed input held in tmpfs and deleted on the pod at 09:24:55Z, server process group killed by exact pid. Transfer: HF_HUB_DISABLE_XET=1 (plain HTTPS, same verification).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 73.40496740812814,
    "I_sealed": 74.71125008803229,
    "base": 74.05810874808022,
    "excess": -11.145809450259854,
    "gap": -1.3062826799041432,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 82.748926388772,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 79.2387543252595,
       "judge": 80.22412656558998,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 68.3768115942029,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 39.99999999999999,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 69.08916424140952,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 78.43444822920543,
       "hard": 75.77623771077789,
       "judge": 66.33819477685424,
       "standard": 55.168829505607754
      }
     },
     "sealed|choice": {
      "cc": 84.69020764774342,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 83.86723719208975,
       "judge": 80.60724779627816,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 72.53036721384638,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 75.86206896551724,
       "hard": 65.45454545454545,
       "judge": 78.46153846153847,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 66.91317540250708,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 86.28980270811569,
       "hard": 61.75286075954152,
       "judge": 67.36429587432818,
       "standard": 66.86881032790222
      }
     }
    }
   },
   "jevbench_score": 74.05642013689533,
   "key": "basal-omni-1.0-ultra",
   "last_measured_on": "2026-10-17",
   "licence": "Apache-2.0 (model card and HF metadata; base google/gemma-4-31B-it Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.28",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Remek/basal-omni-1.0-ultra@b8e214cb42d0a7a4cb2c3a93b0ba02c65d7590e4",
   "noul_decisive": {
    "acc_among_decisive": 0.938953488372093,
    "decisive": 344,
    "decisive_rate": 0.9222520107238605,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.0178774866748,
     "cost": 50.37412598110784,
     "intelligence": 74.05810874808022,
     "speed": 87.31015625661291
    },
    "board_release": "v1.6.43",
    "capability": 82.53799311737751,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 3,
    "capability_rank": 6,
    "composite_ci95": {
     "A": [
      70.13672757527847,
      72.63966201572218
     ],
     "B": [
      69.95644992290508,
      73.66079468239673
     ],
     "C": [
      70.13672757527847,
      72.63966201572218
     ]
    },
    "jevbench_score": 71.69146765135073,
    "measured_in": "v1.6.28",
    "method": "v1.6",
    "open_board_rank": 6,
    "rank": 17,
    "ranks": {
     "A": 17,
     "B": 11,
     "C": 17,
     "capability": 15
    },
    "scores": {
     "A": 71.69146765135073,
     "B": 72.15261728693953,
     "C": 71.69146765135073
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3538213029736653,
     "p50_s_raw": 0.10191065148683265,
     "p95_s_adjusted": 0.5250496784574352,
     "p95_s_raw": 0.1875248392287176
    },
    "views": {
     "Balanced 33:33:33": 66.95263980712231,
     "Emphasis on Accuracy 60:20:20": 69.62468220908855,
     "Emphasis on Cost 20:20:60": 59.16409553572044,
     "Emphasis on Speed 20:60:20": 73.83928140922333,
     "Intelligence only": 74.05810874808022
    }
   },
   "rank": 30,
   "ranked": true,
   "ranks": {
    "A": 30,
    "B": 18,
    "C": 26,
    "capability": 18
   },
   "repo": "https://huggingface.co/Remek/basal-omni-1.0-ultra",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.05642013689533,
    "B": 74.05675785297191,
    "C": 74.05642013689533
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3653893249924295,
    "p50_s_raw": 0.10769466249621473,
    "p95_s_adjusted": 0.594361786666559,
    "p95_s_raw": 0.2221808933332795
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 72.80326542289306,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 18.4097962686792
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 72.80326542289306
      },
      "calibration_gen": {
       "accuracy": 0.9900990099009901,
       "cc": 94.21261550687389,
       "cc_by_type": {
        "choice": 100,
        "noul": 92,
        "score": 90.63784652062168
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 94.21261550687389
      },
      "composite_gen": {
       "accuracy": 0.8811881188118812,
       "cc": 80.10110527775385,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 70.37037037037037,
        "score": 73.18497798321634
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 80.10110527775385
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.29139788288848,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 95.87419364866541
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.29139788288848
      },
      "jaggedness": {
       "accuracy": 0.7833333333333333,
       "cc": 68.09939011803003,
       "cc_by_type": {
        "choice": 71.12608277189607,
        "noul": 67.56756756756756,
        "score": 65.60452001462644
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 68.09939011803003
      },
      "llm_router": {
       "accuracy": 0.9265175718849841,
       "cc": 83.72918389969921,
       "cc_by_type": {
        "choice": 95.24522321035484,
        "noul": 93.81443298969072,
        "score": 62.12789549905209
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.72918389969921
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7037037037037037,
       "cc": 47.98400157140998,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 50,
        "score": 43.43654079670418
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 47.98400157140998
      },
      "multilingual": {
       "accuracy": 0.8072727272727273,
       "cc": 68.0930448969844,
       "cc_by_type": {
        "choice": 77.50244538637104,
        "noul": 58.33333333333333,
        "score": 68.44335597124882
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 68.0930448969844
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 75.63122739536792,
       "cc_by_type": {
        "choice": 100,
        "noul": 53.33333333333334,
        "score": 73.5603488527704
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 75.63122739536792
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.9375,
       "cc": 85.46096801841463,
       "cc_by_type": {
        "choice": 93.993993993994,
        "noul": 100,
        "score": 62.38891006124994
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.46096801841463
      },
      "usecase": {
       "accuracy": 0.7954545454545454,
       "cc": 69.10006746771133,
       "cc_by_type": {
        "choice": 77.94117647058823,
        "noul": 62.5,
        "score": 66.85902593254578
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 69.10006746771133
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 38.9929915779476,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": 5.074212829080893
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 38.9929915779476
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 75.67364841521491,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 47.7582724345848
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 75.67364841521491
      },
      "de": {
       "accuracy": 0.875,
       "cc": 86.07256385800456,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 82.11149688374817
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 86.07256385800456
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.56994661948434,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.56272626441205
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.56994661948434
      },
      "en": {
       "accuracy": 0.8722849695916595,
       "cc": 77.45642365975384,
       "cc_by_type": {
        "choice": 87.47879571887088,
        "noul": 74.82014388489209,
        "score": 70.07033137549857
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 77.45642365975384
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 76.25205214982573,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 38.54408467133693
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 76.25205214982573
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 88.34203473378807,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": 97.81298944726586
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 88.34203473378807
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 84.56323770395797,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 96.04265428834448
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.56323770395797
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 70.7140566079013,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 75,
        "score": 70.6617228963296
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 70.7140566079013
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 62.1449141842809,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 0.46436063546154704
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 62.1449141842809
      },
      "ja": {
       "accuracy": 0.8,
       "cc": 57.6427804823853,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 60.00000000000001,
        "score": 27.225402606806416
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 57.6427804823853
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 42.774593132787025,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 72.76822384280551
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 42.774593132787025
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.44250882836059,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.32752648508176
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.44250882836059
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 10.262858950429482,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -35.87808981537821
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 10.262858950429482
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 65.2787917542296,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 71.0712461894768
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 65.2787917542296
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.14654979605274,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 52.867463177197706
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.14654979605274
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 51.503527144875044,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 19.999999999999996,
        "score": 60.04249632824212
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 51.503527144875044
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 86.36411186126297,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 80.83146601857152
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 86.36411186126297
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.41808992895669,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 94.51233430299908
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.41808992895669
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 29.716048736332187,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 22.481479542329907
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 29.716048736332187
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.884,
       "cc": 85.01693488202989,
       "cc_by_type": {
        "choice": 85.01693488202989
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 85.01693488202989
      },
      "noul": {
       "accuracy": 0.8613333333333333,
       "cc": 72.26666666666665,
       "cc_by_type": {
        "noul": 72.26666666666665
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 72.26666666666665
      },
      "score": {
       "accuracy": 0.7946666666666666,
       "cc": 68.21305822309168,
       "cc_by_type": {
        "score": 68.21305822309168
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 68.21305822309168
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 37.62962300520317,
       "cc_by_type": {
        "choice": 68,
        "noul": 0,
        "score": 44.888869015609515
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 37.62962300520317
      },
      "code_linting": {
       "accuracy": 0.8333333333333334,
       "cc": 83.27418447680438,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 68.45609380308395
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 83.27418447680438
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 63.892557231272725,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100,
        "score": 7.9378342954441905
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 63.892557231272725
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 4.038044257628672,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -51.04376196395608
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 4.038044257628672
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 54.637068852440315,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 74.05613409355283
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 54.637068852440315
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": 22.467454630086888,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 66.66666666666667,
        "score": -61.52845371980222
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 22.467454630086888
      },
      "financial_crime": {
       "accuracy": 0.8,
       "cc": 65.815735202619,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 74.19790588796901
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 65.815735202619
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 46.54917074605785,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0,
        "score": 71.00639725559512
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 46.54917074605785
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 41.202351728419735,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 66.10705518525921
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 41.202351728419735
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 81.35961326233185,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100,
        "score": 56.92680552575358
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 81.35961326233185
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 70.4858410918331,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 45.3076524744657
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 70.4858410918331
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 14.624727606002695,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": 73.98170969972853
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 14.624727606002695
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 59.63569309338237,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 58.907079280147116
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.63569309338237
      },
      "model_routing": {
       "accuracy": 0.9032258064516129,
       "cc": 81.17230996306374,
       "cc_by_type": {
        "choice": 92.98701298701299,
        "noul": 100,
        "score": 50.52991690217823
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.17230996306374
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 26.178743649164545,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -100,
        "score": 78.53623094749362
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 26.178743649164545
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 70.2022507842471,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 77.27341901940797
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 70.2022507842471
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 61.53603210445658,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 100,
        "score": 28.94307168282788
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 61.53603210445658
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 47.3280686936479,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 52.12913361717557
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 47.3280686936479
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.6504026323173,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.95120789695187
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.6504026323173
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.27498047407003,
      91.59920157440426
     ],
     "I": [
      69.12512137353171,
      78.15790301804346
     ],
     "capability": [
      78.44406909592927,
      84.3637194100243
     ],
     "composite": null,
     "seed": 21
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.20661225231586,
     "capability": 82.13236050019805,
     "intelligence": 74.05810874808022
    },
    "per_type": {
     "choice": {
      "accuracy": 0.884,
      "calibration": 90.24757479214279,
      "cc": 85.01693488202989,
      "cc_by_type": {
       "choice": 85.01693488202989
      },
      "cc_tiered": 83.71956701825772,
      "cc_tiered_open": 82.748926388772,
      "cc_tiered_sealed": 84.69020764774342,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 85.01693488202989
     },
     "noul": {
      "accuracy": 0.8613333333333333,
      "calibration": 88.94486578314562,
      "cc": 72.26666666666665,
      "cc_by_type": {
       "noul": 72.26666666666665
      },
      "cc_tiered": 70.45358940402464,
      "cc_tiered_open": 68.3768115942029,
      "cc_tiered_sealed": 72.53036721384638,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 72.26666666666665
     },
     "score": {
      "accuracy": 0.7946666666666666,
      "calibration": 91.42739618165918,
      "cc": 68.21305822309168,
      "cc_by_type": {
       "score": 68.21305822309168
      },
      "cc_tiered": 68.0011698219583,
      "cc_tiered_open": 69.08916424140952,
      "cc_tiered_sealed": 66.91317540250708,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.21305822309168
     }
    },
    "raw_on_A": {
     "calibration": 86.85140384603928,
     "capability": 80.08523448632982,
     "intelligence": 73.31906512662039
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.88574226900883,
    "Emphasis on Accuracy 60:20:20": 71.49697201513041,
    "Emphasis on Cost 20:20:60": 63.540768629753856,
    "Emphasis on Speed 20:60:20": 75.74221337113535,
    "Intelligence only": 74.05810874808022
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Babak Barazandeh (bbarazandeh@cribl.io, GitHub @bbarazandeh-crypto)",
   "axes": {
    "calibration": 82.13857452060611,
    "cost": null,
    "intelligence": 38.67355961154223,
    "speed": 80.91147986788948
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1281668719369918,
      "ece_score": 74.36662561260164,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 74.36662561260164,
      "score_before_time_limit": 74.36662561260164,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.28800000000000003,
      "brier": 0.1440167151699402,
      "ece": 0.07805321715454966,
      "n": 375,
      "score": 84.38935656909007,
      "score_before_time_limit": 84.38935656909007,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9146666666666666,
      "interval90_width": 0.45133333333333325,
      "n": 375,
      "nrps": 0.10596156489230459,
      "score": 87.65974138012665,
      "score_before_time_limit": 87.65974138012665,
      "time_limit_failed_share": 0,
      "top_ece": 0.07042180375258127
     }
    },
    "score": 82.13857452060611
   },
   "capability": 60.40606706607417,
   "class": "Hosted decision API",
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED - NOT A MEASUREMENT FAILURE. The base is disclosed (Qwen/Qwen3.5-4B-Base; reference() resolves it to deepinfra:Qwen/Qwen3.5-4B at 0.03/0.15), but the endpoint reports usage.input_tokens ONLY - no output_tokens on any answer - and the system is API-only, so 'nothing is generated' cannot be verified in its code; the published 02 Oct rule keeps an unverified null unpriced. The frozen scorer therefore gives no Cost axis and no composite; the row is reported on Capability. PRICING REFERENCE NOTE: if the author confirms (or independently verified evidence establishes) a one-pass readout with 0 output tokens, the row prices by one multiplication from the raw at 0.03/M input (receipts/COST-ALTERNATIVES-R59.json).",
    "common_basis": null,
    "kind": "estimated",
    "usage_estimated_rows": 1500,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "BB-Qwen3.5-4B-LoRA (Babak Barazandeh, LoRA on Qwen3.5-4B-Base, hosted /v1/systemone)",
   "endpoint_condition": "Hosted full-set API measured 2026-10-07; submitted LoRA checkpoint-030500 on Qwen3.5-4B-Base, weights not released. Serial requests over author ngrok tunnel, no retry; latency includes transit. Complete 1500/1500 answers. Input usage is reported but output usage is absent, so Cost and composite remain unpriced; Capability is reported. Exposure ledger recorded before dispatch.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "gpu": null,
   "intelligence": {
    "I_open": 40.54310423897533,
    "I_sealed": 36.80401498410914,
    "base": 38.67355961154223,
    "excess": 1.16570161242795,
    "gap": 3.739089254866194,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 54.16068441724518,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 52.815350739226176,
       "judge": 64.40342781806196,
       "standard": 32.18920050230222
      }
     },
     "open|noul": {
      "cc": 28.017253278122844,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 39.45137502155798,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 35.5237907386309,
       "hard": 38.20598063967837,
       "judge": 35.150958941159246,
       "standard": 50.35658004737882
      }
     },
     "sealed|choice": {
      "cc": 53.65959260673962,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 68.07331313280615,
       "hard": 49.09502262443438,
       "judge": 50.65176908752329,
       "standard": 60.09360758714129
      }
     },
     "sealed|noul": {
      "cc": 19.15862307210663,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 56.00000000000001,
       "hard": -4.854368932038833,
       "judge": 26.315789473684205,
       "standard": 38.0281690140845
      }
     },
     "sealed|score": {
      "cc": 37.59382927348118,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 61.46292444619053,
       "hard": 28.220009445462267,
       "judge": 41.5085830641898,
       "standard": 38.53479065710138
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "bb-qwen3.5-4b-lora",
   "last_measured_on": "2026-10-07",
   "licence": null,
   "listing": "listed",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "checkpoint-030500 (author declaration); base revision 1001bb4d",
   "not_ranked_because": "registry ranked=false or no composite",
   "noul_decisive": {
    "acc_among_decisive": 0.8764044943820225,
    "decisive_rate": 0.712,
    "supported": true,
    "valid": 375
   },
   "open": "closed weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.13857452060611,
     "cost": null,
     "intelligence": 38.67355961154223,
     "speed": 81.57813729165589
    },
    "board_release": "v1.6.43",
    "capability": 60.40606706607417,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "capability": 65
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.6954362019896507,
     "p50_s_raw": 0.6954362019896507,
     "p95_s_adjusted": 0.9998364746570587,
     "p95_s_raw": 0.9998364746570587
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": 23.13675425207104
    }
   },
   "provenance": {
    "carried_metadata_from": "Current registry and GitHub #156; no credential or private endpoint URL copied",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "capability": 65
   },
   "repo": "https://github.com/fstandhartinger/jevbench/issues/156",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.7166377417743206,
    "p50_s_raw": 0.7166377417743206,
    "p95_s_adjusted": 1.131233705580234,
    "p95_s_raw": 1.131233705580234
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B-Base",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.70685263219829,
      84.35075129046632
     ],
     "I": [
      32.59884752255904,
      43.02764829916808
     ],
     "capability": [
      55.59198238994904,
      62.87426662252782
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.13857452060611,
     "capability": 60.40606706607417,
     "intelligence": 38.67355961154223
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6493333333333333,
      "calibration": 74.36662561260164,
      "cc": 55.15474171821564,
      "cc_by_type": {
       "choice": 55.15474171821564
      },
      "cc_tiered": 53.9101385119924,
      "cc_tiered_open": 54.16068441724518,
      "cc_tiered_sealed": 53.65959260673962,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 55.15474171821564
     },
     "noul": {
      "accuracy": 0.624,
      "calibration": 84.38935656909007,
      "cc": 24.8,
      "cc_by_type": {
       "noul": 24.8
      },
      "cc_tiered": 23.587938175114736,
      "cc_tiered_open": 28.017253278122844,
      "cc_tiered_sealed": 19.15862307210663,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.8
     },
     "score": {
      "accuracy": 0.5413333333333333,
      "calibration": 87.65974138012665,
      "cc": 38.101218047454275,
      "cc_by_type": {
       "score": 38.101218047454275
      },
      "cc_tiered": 38.52260214751958,
      "cc_tiered_open": 39.45137502155798,
      "cc_tiered_sealed": 37.59382927348118,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 38.101218047454275
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": 23.13675425207104
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bespoke Labs",
   "axes": {
    "calibration": 77.04569964036838,
    "cost": 65.75776727887356,
    "intelligence": 56.54011796945148,
    "speed": 89.00367576375322
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1589499873699225,
      "ece_score": 68.2100025260155,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 68.2100025260155,
      "score_before_time_limit": 68.2100025260155,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.13477088948787064,
      "brier": 0.12391951730825534,
      "ece": 0.10295952013212975,
      "n": 371,
      "score": 79.40809597357405,
      "score_before_time_limit": 79.40809597357405,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8699186991869918,
      "interval90_width": 0.20505871725383928,
      "n": 369,
      "nrps": 0.07833002311655612,
      "score": 83.51900042151561,
      "score_before_time_limit": 83.51900042151561,
      "time_limit_failed_share": 0,
      "top_ece": 0.1256449842265658
     }
    },
    "score": 77.04569964036838
   },
   "capability": 66.79290880490993,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 925,157 input + 0 output tokens over 1,477 decisions = 626.375761679 tokens/decision. GPU-time cost USD 0.0138486601398688/1,000; old list-price cost USD 0.0626375761679079/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0138486601398688/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.06167713333333334
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013848660139868772
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bespoke Nimble 9B v3 (Qwen3.5-9B LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 57.07198447174714,
    "I_sealed": 56.008251467155816,
    "base": 56.54011796945148,
    "excess": -1.5096546378469164,
    "gap": 1.0637330045913274,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 72.11673425629002,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 60.36489462094999,
       "judge": 76.26895187870797,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 48.59075224292616,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 39.99999999999999,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 50.508466916025256,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.68883649626082,
       "hard": 50.44357111216904,
       "judge": 56.6073538275629,
       "standard": 29.899743366313437
      }
     },
     "sealed|choice": {
      "cc": 63.08410687973317,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 77.38526346907102,
       "hard": 58.76696832579185,
       "judge": 60.89385474860336,
       "standard": 67.8531838896416
      }
     },
     "sealed|noul": {
      "cc": 51.038414647311576,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": 28.155339805825253,
       "judge": 65.78947368421053,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 53.90223287442272,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 75.56838262623131,
       "hard": 43.523681374236745,
       "judge": 58.275920598763435,
       "standard": 57.26572941237931
      }
     }
    }
   },
   "jevbench_score": 70.04188420031245,
   "key": "bespoke-nimble-9b-v3",
   "last_measured_on": "2026-10-06",
   "licence": "cc-by-nc-4.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "bespokelabs/Bespoke-Nimble-9B-v3@8e927b9b4afdbb14479fac10a7364d1a695be208",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8940809968847352,
    "decisive_rate": 0.8652291105121294,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.04569964036838,
     "cost": 46.094951684283686,
     "intelligence": 56.54011796945148,
     "speed": 89.19502471208428
    },
    "board_release": "v1.6.43",
    "capability": 66.79290880490993,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 34,
    "capability_rank": 55,
    "composite_ci95": {
     "A": [
      51.086235675343055,
      54.820858958312876
     ],
     "B": [
      49.08081410151091,
      54.214427134732105
     ],
     "C": [
      35.737762455481224,
      54.819952316161036
     ]
    },
    "jevbench_score": 53.47656993239491,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 39,
    "rank": 63,
    "ranks": {
     "A": 63,
     "B": 60,
     "C": 56,
     "capability": 94
    },
    "scores": {
     "A": 53.47656993239491,
     "B": 52.29615241766553,
     "C": 47.486970389641556
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.33777692201547327,
     "p50_s_raw": 0.09388846100773662,
     "p95_s_adjusted": 0.3563424444932025,
     "p95_s_raw": 0.10317122224660125
    },
    "views": {
     "Balanced 33:33:33": 50.39687437311006,
     "Emphasis on Accuracy 60:20:20": 49.43254820883513,
     "Emphasis on Cost 20:20:60": 45.21647399422632,
     "Emphasis on Speed 20:60:20": 58.20016580649009,
     "Intelligence only": 48.05332347466076
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 78,
   "ranked": true,
   "ranks": {
    "A": 78,
    "B": 80,
    "C": 82,
    "capability": 120
   },
   "repo": "https://huggingface.co/bespokelabs/Bespoke-Nimble-9B-v3",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.04188420031245,
    "B": 66.8491710856597,
    "C": 62.196900161318645
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3390120119787753,
    "p50_s_raw": 0.09450600598938763,
    "p95_s_adjusted": 0.37103710017108815,
    "p95_s_raw": 0.11051855008554409
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-9B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.11138007423693,
      79.85549338743986
     ],
     "I": [
      49.993162280566764,
      61.190765007174605
     ],
     "capability": [
      61.91008408542518,
      69.96614101398373
     ],
     "composite": null,
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.04569964036838,
     "capability": 66.79290880490993,
     "intelligence": 56.54011796945148
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7333333333333333,
      "calibration": 68.2100025260155,
      "cc": 65.89714199103851,
      "cc_by_type": {
       "choice": 65.89714199103851
      },
      "cc_tiered": 67.6004205680116,
      "cc_tiered_open": 72.11673425629002,
      "cc_tiered_sealed": 63.08410687973317,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.89714199103851
     },
     "noul": {
      "accuracy": 0.7653333333333333,
      "calibration": 79.40809597357405,
      "cc": 53.06666666666666,
      "cc_by_type": {
       "noul": 53.06666666666666
      },
      "cc_tiered": 49.81458344511887,
      "cc_tiered_open": 48.59075224292616,
      "cc_tiered_sealed": 51.038414647311576,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 53.06666666666666
     },
     "score": {
      "accuracy": 0.6586666666666666,
      "calibration": 83.51900042151561,
      "cc": 53.90613650563776,
      "cc_by_type": {
       "score": 53.90613650563776
      },
      "cc_tiered": 52.20534989522399,
      "cc_tiered_open": 50.508466916025256,
      "cc_tiered_sealed": 53.90223287442272,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.90613650563776
     }
    },
    "raw_on_A": {
     "calibration": 79.9777663718046,
     "capability": 70.03747268625314,
     "intelligence": 60.09717900070167
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 67.98192557737386,
    "Emphasis on Accuracy 60:20:20": 62.891111979126606,
    "Emphasis on Cost 20:20:60": 67.07444834943784,
    "Emphasis on Speed 20:60:20": 75.07467072410668,
    "Intelligence only": 56.54011796945149
   }
  },
  {
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Reza Sayar",
   "axes": {
    "calibration": 88.50232899798883,
    "cost": 55.738539603831974,
    "intelligence": 43.77513601921869,
    "speed": 72.92528840336786
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08046726298491785,
      "ece_score": 83.90654740301643,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 83.90654740301643,
      "score_before_time_limit": 83.90654740301643,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3045822102425876,
      "brier": 0.11954030846459725,
      "ece": 0.04318003176619355,
      "n": 371,
      "score": 91.36399364676129,
      "score_before_time_limit": 91.36399364676129,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.964769647696477,
      "interval90_width": 0.4652213188798555,
      "n": 369,
      "nrps": 0.08597620729262767,
      "score": 90.23644594418879,
      "score_before_time_limit": 90.23644594418879,
      "time_limit_failed_share": 0,
      "top_ece": 0.0546474369117983
     }
    },
    "score": 88.50232899798883
   },
   "capability": 66.13873250860377,
   "class": "system-one-open",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 942,533 input + 1,624 output tokens over 1,624 decisions = 581.377463054 tokens/decision. GPU-time cost USD 0.0298800977022063/1,000; old list-price cost USD 0.246758534482759/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0298800977022063/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02988009770220635
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bev / Bonsai 27B",
   "endpoint_condition": "evaluator-owned H100 GPU",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 46.71700087936099,
    "I_sealed": 40.833271159076396,
    "base": 43.77513601921869,
    "excess": 3.310342077846348,
    "gap": 5.883729720284592,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 63.31036599788361,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 47.153192827933324,
       "judge": 72.31377719182596,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 30.23947550034507,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 46.6011611398543,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.472258307241056,
       "hard": 30.60094415356245,
       "judge": 62.977497036063276,
       "standard": 48.101542684431145
      }
     },
     "sealed|choice": {
      "cc": 56.58157537232299,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 66.74303451333975,
       "hard": 49.09502262443438,
       "judge": 53.44506517690876,
       "standard": 71.17871659071314
      }
     },
     "sealed|noul": {
      "cc": 20.312021130359053,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 52,
       "hard": -0.9708737864077666,
       "judge": 26.315789473684205,
       "standard": 38.0281690140845
      }
     },
     "sealed|score": {
      "cc": 45.60621697454716,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 67.9108636782132,
       "hard": 33.06846017805201,
       "judge": 52.300237792599205,
       "standard": 49.48837598862635
      }
     }
    }
   },
   "jevbench_score": 46.598366686554485,
   "key": "bev-bonsai-27b",
   "last_measured_on": "2026-10-04",
   "licence": "Not independently recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9147286821705426,
    "decisive_rate": 0.6954177897574124,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.50232899798883,
     "cost": 28.231834531826323,
     "intelligence": 43.77513601921869,
     "speed": 72.98222461463935
    },
    "board_release": "v1.6.43",
    "capability": 66.13873250860377,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      8.55065455714213,
      14.587291870035246
     ],
     "B": [
      8.22423019498041,
      14.527264406104972
     ],
     "C": [
      5.9379545535709255,
      10.130063798635586
     ]
    },
    "jevbench_score": 11.7393704396132,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 125,
    "rank": 164,
    "ranks": {
     "A": 164,
     "B": 163,
     "C": 165,
     "capability": 97
    },
    "scores": {
     "A": 11.7393704396132,
     "B": 11.515061943796129,
     "C": 8.152340583064722
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 2.1222116896882652,
     "p50_s_raw": 0.9861058448441327,
     "p95_s_adjusted": 2.371312748454511,
     "p95_s_raw": 1.1106563742272555
    },
    "views": {
     "Balanced 33:33:33": 10.186877852314403,
     "Emphasis on Accuracy 60:20:20": 10.385148364433647,
     "Emphasis on Cost 20:20:60": 8.555937486887114,
     "Emphasis on Speed 20:60:20": 12.296000304326956,
     "Intelligence only": 10.697460589059927
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 126,
   "ranked": true,
   "ranks": {
    "A": 126,
    "B": 126,
    "C": 131,
    "capability": 126
   },
   "repo": "https://huggingface.co/Reza2kn/Bev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 46.598366686554485,
    "B": 43.236606632674714,
    "C": 32.35997686566283
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 2.127405576966703,
    "p50_s_raw": 0.9887027884833515,
    "p95_s_adjusted": 2.3967396812047808,
    "p95_s_raw": 1.1233698406023904
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Reza2kn/Bev @ f01edf952e3f2da8147aa08ec6d6229c863d5d19. Unchanged Prism ML Ternary-Bonsai-2-27B PQ2_0 quantization of Qwen3.8-27B, with selected-token readout; not a fine-tune. GGUF SHA-256 3907dc1658db1f78a9826bf8d5bcb8dc65db0d466388937af57f2294fae62ec1. H100; CUDA backend compiled from the author-pinned source 9a9394a895b96003ca842a6041cb28ac49a108f7 with the author patch, rather than their prebuilt binary. The wire shim drops noul criteria because the submitted schema rejects them; that loses option descriptions for noul.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.75270117761126,
      88.77503324811279
     ],
     "I": [
      38.25187875678337,
      48.17301489659999
     ],
     "capability": [
      60.910547743797885,
      67.5609085693478
     ],
     "composite": null,
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.50232899798883,
     "capability": 66.13873250860377,
     "intelligence": 43.77513601921869
    },
    "per_type": {
     "choice": {
      "accuracy": 0.672,
      "calibration": 83.90654740301643,
      "cc": 58.053484648977374,
      "cc_by_type": {
       "choice": 58.053484648977374
      },
      "cc_tiered": 59.9459706851033,
      "cc_tiered_open": 63.31036599788361,
      "cc_tiered_sealed": 56.58157537232299,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 58.053484648977374
     },
     "noul": {
      "accuracy": 0.6293333333333333,
      "calibration": 91.36399364676129,
      "cc": 25.86666666666666,
      "cc_by_type": {
       "noul": 25.86666666666666
      },
      "cc_tiered": 25.27574831535206,
      "cc_tiered_open": 30.23947550034507,
      "cc_tiered_sealed": 20.312021130359053,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 25.86666666666666
     },
     "score": {
      "accuracy": 0.6,
      "calibration": 90.23644594418879,
      "cc": 45.43825770414722,
      "cc_by_type": {
       "score": 45.43825770414722
      },
      "cc_tiered": 46.103689057200725,
      "cc_tiered_open": 46.6011611398543,
      "cc_tiered_sealed": 45.60621697454716,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 45.43825770414722
     }
    },
    "raw_on_A": {
     "calibration": 80.9895865212274,
     "capability": 63.89601132732785,
     "intelligence": 46.8024361334283
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 42.19480220360983,
    "Emphasis on Accuracy 60:20:20": 38.25424068205824,
    "Emphasis on Cost 20:20:60": 42.40484911181807,
    "Emphasis on Speed 20:60:20": 46.78208638571798,
    "Intelligence only": 33.553861221017264
   }
  },
  {
   "adapter_id": "images/bge-reranker-v2-m3/spec.json:reranker_adapter.RerankerAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "BAAI",
   "axes": {
    "calibration": 88.10046932372114,
    "cost": 59.325605115070715,
    "intelligence": 0.06512222845080358,
    "speed": 93.5678736531519
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.027473000546109336,
      "ece_score": 94.50539989077814,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 94.50539989077814,
      "score_before_time_limit": 94.50539989077814,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 1,
      "brier": 0.25096294008809744,
      "ece": 0.0684901503152521,
      "n": 375,
      "score": 86.30196993694959,
      "score_before_time_limit": 86.30196993694959,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 1,
      "interval90_width": 1,
      "n": 375,
      "nrps": 0.1745560185107233,
      "score": 83.49403814343572,
      "score_before_time_limit": 83.49403814343572,
      "time_limit_failed_share": 0,
      "top_ece": 0.07778160931028115
     }
    },
    "score": 88.10046932372114
   },
   "capability": 44.08279577608597,
   "class": "reranker",
   "composite_ci95": {
    "A": [
     0,
     0.008573175268890313
    ],
    "B": [
     0,
     0.005538532607600911
    ],
    "C": [
     0,
     0.0059535939367293825
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: base-model market reference (deepinfra:BAAI/bge-m3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.022688885467980296
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "BAAI bge-reranker-v2-m3",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 0.018692443590466585,
    "I_sealed": 0.11155201331114058,
    "base": 0.06512222845080358,
    "excess": -2.666247212158918,
    "gap": -0.092859569720674,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 0.05607733077139976,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486491,
       "hard": 7.518087448883308,
       "judge": -14.700065919578117,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 5.306919494288776,
       "hard": -1.440659315203363,
       "judge": -2.980317227364382,
       "standard": -2.397579483854617
      }
     },
     "sealed|choice": {
      "cc": 0.33465603993342175,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -5.092010937846398,
       "hard": -0.791855203619931,
       "judge": 2.234636871508399,
       "standard": 2.451040768567572
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 2.319622122935927,
       "hard": -0.38388489349363564,
       "judge": 0.07005074041354087,
       "standard": -0.8557298803223867
      }
     }
    }
   },
   "jevbench_score": 4.4076704742616216e-07,
   "key": "bge-reranker-v2-m3",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive_rate": 0,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.10046932372114,
     "cost": 59.325605115070715,
     "intelligence": 0.06512222845080358,
     "speed": 93.86976236118387
    },
    "board_release": "v1.6.43",
    "capability": 44.08279577608597,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 102,
    "capability_rank": 130,
    "composite_ci95": {
     "A": [
      0,
      0.008573668302290527
     ],
     "B": [
      0,
      0.005538697219146524
     ],
     "C": [
      0,
      0.005953936321035087
     ]
    },
    "jevbench_score": 4.4076803151681546e-07,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 194,
    "rank": 238,
    "ranks": {
     "A": 238,
     "B": 238,
     "C": 238,
     "capability": 191
    },
    "scores": {
     "A": 4.4076803151681546e-07,
     "B": 2.758281497029271e-07,
     "C": 3.060889107755663e-07
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1841465687379241,
     "p50_s_raw": 0.01707328436896205,
     "p95_s_adjusted": 0.2227717584464699,
     "p95_s_raw": 0.03638587922323495
    },
    "views": {
     "Balanced 33:33:33": 3.308199423158738e-07,
     "Emphasis on Accuracy 60:20:20": 1.8400822603920523e-07,
     "Emphasis on Cost 20:20:60": 5.501608977147286e-07,
     "Emphasis on Speed 20:60:20": 5.506039727293195e-07,
     "Intelligence only": 1.1047086427994778e-07
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 279,
   "ranked": true,
   "ranks": {
    "A": 279,
    "B": 279,
    "C": 279,
    "capability": 230
   },
   "repo": "https://huggingface.co/BAAI/bge-reranker-v2-m3",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 4.4076704742616216e-07,
    "B": 2.758278413965028e-07,
    "C": 3.060882273792793e-07
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18669292386621236,
    "p50_s_raw": 0.018346461933106184,
    "p95_s_adjusted": 0.23555090506561097,
    "p95_s_raw": 0.042775452532805486
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "568M multilingual cross-encoder",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.14061191725726,
      90.57274307882632
     ],
     "I": [
      0,
      1.7895886627489592
     ],
     "capability": [
      41.80444040346114,
      45.59879463409008
     ],
     "composite": {
      "A": [
       0,
       0.008573175268890313
      ],
      "B": [
       0,
       0.005538532607600911
      ],
      "C": [
       0,
       0.0059535939367293825
      ]
     },
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.10046932372114,
     "capability": 44.08279577608597,
     "intelligence": 0.06512222845080358
    },
    "per_type": {
     "choice": {
      "accuracy": 0.22133333333333333,
      "calibration": 94.50539989077814,
      "cc": 0.4196546138324576,
      "cc_by_type": {
       "choice": 0.4196546138324576
      },
      "cc_tiered": 0.19536668535241075,
      "cc_tiered_open": 0.05607733077139976,
      "cc_tiered_sealed": 0.33465603993342175,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0.4196546138324576
     },
     "noul": {
      "accuracy": 0,
      "calibration": 86.30196993694959,
      "cc": -100,
      "cc_by_type": {
       "noul": -100
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2826666666666667,
      "calibration": 83.49403814343572,
      "cc": -0.18199548701309798,
      "cc_by_type": {
       "score": -0.18199548701309798
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 85.43472005366168,
     "capability": 43.01708899298245,
     "intelligence": 0.5994579323032229
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 3.308192031583021e-07,
    "Emphasis on Accuracy 60:20:20": 1.8400808883101353e-07,
    "Emphasis on Cost 20:20:60": 5.501596711672631e-07,
    "Emphasis on Speed 20:60:20": 5.506002871741509e-07,
    "Intelligence only": 1.1047086427994778e-07
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "thegovind",
   "axes": {
    "calibration": 87.79893140350136,
    "cost": 78.06391380283692,
    "intelligence": 35.035615417279956,
    "speed": 91.21351546851116
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06420474972972842,
      "ece_score": 87.1590500540543,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.1590500540543,
      "score_before_time_limit": 87.1590500540543,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.4026666666666666,
      "brier": 0.13994335801090524,
      "ece": 0.06618791977957074,
      "n": 375,
      "score": 86.76241604408585,
      "score_before_time_limit": 86.76241604408585,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.936,
      "interval90_width": 0.5479999999999998,
      "n": 375,
      "nrps": 0.11212365750633133,
      "score": 89.47532811236393,
      "score_before_time_limit": 89.95508858481628,
      "time_limit_failed_share": 0.005333333333333333,
      "top_ece": 0.04438728539867154
     }
    },
    "score": 87.79893140350136
   },
   "capability": 61.417273410390656,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 813,840 input + 0 output tokens over 1,485 decisions = 548.04040404 tokens/decision. GPU-time cost USD 0.00538521284776195/1,000; old list-price cost USD 0.0164412121212121/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00538521284776195/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.03988590000000001
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005385212847761947
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 2,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "blink-4b (thegovind, Qwen3.5-4B derivative)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own serve.py (one forward pass, native probabilities, T 1.0).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 42.02146053059893,
    "I_sealed": 28.049770303960976,
    "base": 35.035615417279956,
    "excess": 4.1321634562822425,
    "gap": 13.971690226637953,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 61.767054755356924,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 56.59012268008808,
       "judge": 68.35860250494397,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 23.933747412008287,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 21.739130434782616,
       "judge": 0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 40.36357942443159,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.57143972802311,
       "hard": 43.497884292247115,
       "judge": 35.671826187924616,
       "standard": 35.02866939176526
      }
     },
     "sealed|choice": {
      "cc": 58.87841680582105,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 77.12833545108006,
       "hard": 52.1221232797502,
       "judge": 60.33300685602351,
       "standard": 61.08415946002954
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 27.586206896551737,
       "hard": -1.8181818181818188,
       "judge": -23.076923076923073,
       "standard": 10.447761194029859
      }
     },
     "sealed|score": {
      "cc": 25.27089410606189,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 42.91358355022767,
       "hard": 15.290430139380662,
       "judge": 28.4348604043804,
       "standard": 31.664527869863658
      }
     }
    }
   },
   "jevbench_score": 30.829294674656747,
   "key": "blink-4b",
   "last_measured_on": "2026-10-16",
   "licence": "other (weights non-commercial, research and evaluation only)",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "thegovind/blink-4b@c243abdd2d79478db510dcdc60a88087c18547c0",
   "noul_decisive": {
    "acc_among_decisive": 0.9017857142857143,
    "decisive": 224,
    "decisive_rate": 0.5973333333333334,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.95885156098548,
     "cost": 63.5219850233163,
     "intelligence": 35.29148713463226,
     "speed": 91.30713992573452
    },
    "board_release": "v1.6.43",
    "capability": 61.62516934780887,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 57,
    "capability_rank": 82,
    "composite_ci95": {
     "A": [
      20.383405879466594,
      38.43953383528232
     ],
     "B": [
      17.414859604087905,
      34.33646076389896
     ],
     "C": [
      14.155142971851804,
      26.694120718946053
     ]
    },
    "jevbench_score": 30.012225474173363,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 69,
    "rank": 101,
    "ranks": {
     "A": 101,
     "B": 102,
     "C": 106,
     "capability": 126
    },
    "scores": {
     "A": 30.012225474173363,
     "B": 26.29431640797227,
     "C": 20.841823245953726
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.264093909971416,
     "p50_s_raw": 0.057046954985708,
     "p95_s_adjusted": 0.28023838466033335,
     "p95_s_raw": 0.06511919233016669
    },
    "views": {
     "Balanced 33:33:33": 27.15945688766263,
     "Emphasis on Accuracy 60:20:20": 22.3004177143674,
     "Emphasis on Cost 20:20:60": 28.79236405729956,
     "Emphasis on Speed 20:60:20": 32.378048784724925,
     "Intelligence only": 17.58206451384755
    }
   },
   "rank": 163,
   "ranked": true,
   "ranks": {
    "A": 163,
    "B": 166,
    "C": 165,
    "capability": 159
   },
   "repo": "https://huggingface.co/thegovind/blink-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 30.829294674656747,
    "B": 26.61299768233994,
    "C": 21.409232412956072
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2652606196701527,
    "p50_s_raw": 0.05763030983507633,
    "p95_s_adjusted": 0.28508585742674764,
    "p95_s_raw": 0.06754292871337383
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6363636363636364,
       "cc": 31.2572362329405,
       "cc_by_type": {
        "choice": 41.666666666666664,
        "noul": 100,
        "score": -47.89495796784515
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 31.2572362329405
      },
      "calibration_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 14.160560852789573,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": -12,
        "score": 14.927069620934985
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 14.160560852789573
      },
      "composite_gen": {
       "accuracy": 0.45544554455445546,
       "cc": 6.709160275028836,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": -33.333333333333336,
        "score": 2.2413019632978926
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 6.709160275028836
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 82.52546910827105,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 87.57640732481315
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 82.52546910827105
      },
      "jaggedness": {
       "accuracy": 0.4444444444444444,
       "cc": 8.81363787222621,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": -35.13513513513513,
        "score": 19.32388320802163
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 8.81363787222621
      },
      "llm_router": {
       "accuracy": 0.7603833865814696,
       "cc": 59.70781779653648,
       "cc_by_type": {
        "choice": 68.3014880690323,
        "noul": 62.88659793814433,
        "score": 47.935367382432815
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 59.70781779653648
      },
      "long_state": {
       "accuracy": 0.2,
       "cc": -103.14840833040694,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": -100,
        "score": -219.12264434605956
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -7.038566180372126,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -50,
        "score": 3.111105582594964
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6509090909090909,
       "cc": 34.3267274057918,
       "cc_by_type": {
        "choice": 74.5679817411151,
        "noul": -11.111111111111116,
        "score": 39.52331158737141
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 34.3267274057918
      },
      "negation_pair_gen": {
       "accuracy": 0.45348837209302323,
       "cc": -10.429069637362135,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": -33.333333333333336,
        "score": -44.620542245419735
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.7708333333333334,
       "cc": 70.70572569906791,
       "cc_by_type": {
        "choice": 70.70572569906791
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 70.70572569906791
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 69.14502272926966,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 80,
        "score": 60.46810122084199
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 69.14502272926966
      },
      "usecase": {
       "accuracy": 0.5272727272727272,
       "cc": 24.832189535408105,
       "cc_by_type": {
        "choice": 57.10784313725491,
        "noul": -8.333333333333337,
        "score": 25.72205880230274
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 24.832189535408105
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 14.05020739233215,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50,
        "score": -12.611282584908313
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 14.05020739233215
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 32.03750075074918,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 29.445835585580895
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 32.03750075074918
      },
      "de": {
       "accuracy": 0.875,
       "cc": 78.73802460999639,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 75,
        "score": 69.17867559990066
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 78.73802460999639
      },
      "el": {
       "accuracy": 0.4,
       "cc": 21.81902108470916,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.16283606604554
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.81902108470916
      },
      "en": {
       "accuracy": 0.5855777584708949,
       "cc": 30.62112327914024,
       "cc_by_type": {
        "choice": 56.50529039186729,
        "noul": 8.633093525179847,
        "score": 26.72498592037359
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 30.62112327914024
      },
      "es": {
       "accuracy": 0.9354838709677419,
       "cc": 73.23800349868046,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 19.714010496041368
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.23800349868046
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -1.5898803127438204,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -33.333333333333336,
        "score": 59.71123337870843
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 39.133701489451745,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 59.754045644825815
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 39.133701489451745
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": 1.584927562947776,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -50,
        "score": -28.48544077484383
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 1.584927562947776
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 65.5019192265144,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -3.4942423204567863
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 65.5019192265144
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 33.175342862764445,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 36.71427322689542
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 33.175342862764445
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 7.694758956980356,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 23.084276870941068
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 7.694758956980356
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.56344076684583,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.3569889672042
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.56344076684583
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -5.87020332999126,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 34.337441958078166
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 25,
       "cc_by_type": {
        "choice": 100,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 25
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 30.78113910343191,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": -19.999999999999996,
        "score": 19.96085277368973
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 30.78113910343191
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 51.433523055552534,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 60.00000000000001,
        "score": 47.33533640295694
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 51.433523055552534
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 22.582548229673375,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -19.999999999999996,
        "score": 38.81147447625417
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 22.582548229673375
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 27.12658273445894,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0,
        "score": 3.118878638159428
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 27.12658273445894
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 86.80980266197905,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 60.42940798593717
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 86.80980266197905
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 5.468607851002157,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -50.260843113660194
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 5.468607851002157
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6893333333333334,
       "cc": 59.872940546126024,
       "cc_by_type": {
        "choice": 59.872940546126024
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 59.872940546126024
      },
      "noul": {
       "accuracy": 0.5386666666666666,
       "cc": 7.733333333333325,
       "cc_by_type": {
        "noul": 7.733333333333325
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 7.733333333333325
      },
      "score": {
       "accuracy": 0.49333333333333335,
       "cc": 29.087528327138855,
       "cc_by_type": {
        "score": 29.087528327138855
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 29.087528327138855
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": 0.8544195377787412,
       "cc_by_type": {
        "choice": 52,
        "noul": -50,
        "score": 0.5632586133362261
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.8544195377787412
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 13.382698260770676,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": 3.5711451618841483
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 13.382698260770676
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 55.70629881232147,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 49.639221640216434
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 55.70629881232147
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -34.52023806987325,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100,
        "score": -9.993462747631444
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 16.29233157470966,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 40.18134255021594
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 16.29233157470966
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -37.46002141648891,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": -100,
        "score": -99.80144789726546
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 25.066909912950777,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 18.898208730448964
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 25.066909912950777
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -15.544132002325737,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100,
        "score": 26.538335700339864
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 11.219886399291537,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -60,
        "score": 14.492992531207937
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 11.219886399291537
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 0.3765944167613687,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": -33.333333333333336,
        "score": -1.2970547225924056
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.3765944167613687
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 30.933446596877317,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 11.111111111111116,
        "score": 4.945042633009211
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 30.933446596877317
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -21.07845262991218,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": 1.2807711425215196
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 21.659522931301932,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": 31.04382791098833
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 21.659522931301932
      },
      "model_routing": {
       "accuracy": 0.7603686635944701,
       "cc": 59.538749649418754,
       "cc_by_type": {
        "choice": 70.7792207792208,
        "noul": 75.75757575757575,
        "score": 32.07945241145974
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 59.538749649418754
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": -3.132727410408895,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 25.150762106585223
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 53.989306990500346,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 45.34767956017328
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 53.989306990500346
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 26.166164438071842,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0,
        "score": 22.83346868367365
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 26.166164438071842
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 4.004736111390193,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -66.66666666666667,
        "score": 22.15913587040247
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 4.004736111390193
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.20773397171418,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 82.62320191514256
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.20773397171418
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.51478120471185,
      88.2252753118533
     ],
     "I": [
      29.913688363668427,
      38.865369340474636
     ],
     "capability": [
      56.71722361866542,
      62.745102389387604
     ],
     "composite": null,
     "seed": 22
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.79893140350136,
     "capability": 61.417273410390656,
     "intelligence": 35.035615417279956
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6893333333333334,
      "calibration": 87.1590500540543,
      "cc": 59.872940546126024,
      "cc_by_type": {
       "choice": 59.872940546126024
      },
      "cc_tiered": 60.322735780588985,
      "cc_tiered_open": 61.767054755356924,
      "cc_tiered_sealed": 58.87841680582105,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.872940546126024
     },
     "noul": {
      "accuracy": 0.5386666666666666,
      "calibration": 86.76241604408585,
      "cc": 7.733333333333325,
      "cc_by_type": {
       "noul": 7.733333333333325
      },
      "cc_tiered": 11.966873706004144,
      "cc_tiered_open": 23.933747412008287,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 7.733333333333325
     },
     "score": {
      "accuracy": 0.49333333333333335,
      "calibration": 89.47532811236393,
      "cc": 29.087528327138855,
      "cc_by_type": {
       "score": 29.087528327138855
      },
      "cc_tiered": 32.81723676524674,
      "cc_tiered_open": 40.36357942443159,
      "cc_tiered_sealed": 25.27089410606189,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 29.087528327138855
     }
    },
    "raw_on_A": {
     "calibration": 84.82814231125586,
     "capability": 60.24380895524264,
     "intelligence": 35.659475599229424
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 28.15586053346972,
    "Emphasis on Accuracy 60:20:20": 22.440387994496948,
    "Emphasis on Cost 20:20:60": 31.500173228409462,
    "Emphasis on Speed 20:60:20": 33.067266395468394,
    "Intelligence only": 17.20240795670602
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Pixilab",
   "axes": {
    "calibration": 90.67498392881862,
    "cost": 71.06457524185485,
    "intelligence": 49.21581563112197,
    "speed": 91.13697610522254
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04822184846568736,
      "ece_score": 90.35563030686254,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 90.35563030686254,
      "score_before_time_limit": 90.35563030686254,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3296703296703297,
      "brier": 0.10252744982113196,
      "ece": 0.049136862184262595,
      "n": 364,
      "score": 90.17262756314747,
      "score_before_time_limit": 90.17262756314747,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9483695652173914,
      "interval90_width": 0.3299365942028985,
      "n": 368,
      "nrps": 0.07594938571288598,
      "score": 91.49669391644588,
      "score_before_time_limit": 91.49669391644588,
      "time_limit_failed_share": 0,
      "top_ece": 0.047058367979098334
     }
    },
    "score": 90.67498392881862
   },
   "capability": 69.9453997799703,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 791,073 input + 1,471 output tokens over 1,500 decisions = 528.362666667 tokens/decision. GPU-time cost USD 0.00921539931730195/1,000; old list-price cost USD 0.04775858/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00921539931730195/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00921539931730195
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Blink v0.3 26B-A4B",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX 5090 32 GB pod (Drobeta-Turnu Severin, Romania), offline after setup (egress cut and proven), plain vLLM 0.30.0 with the model card's own serve command plus ONE serving-only deviation: VLLM_BATCH_INVARIANT=1 (needed to pass the 0.005 repeatability gate; see DETERMINISM.md), served through our loopback shim implementing the card's reference decide() client (T=1.3, softmax over option-letter logprobs); context 32,768 (29 of 1,500 states are 77k-82k tokens and were refused by vLLM = recorded misses)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 53.29629960522241,
    "I_sealed": 45.135331657021524,
    "base": 49.21581563112197,
    "excess": -1.9122766331388288,
    "gap": 8.160967948200884,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 74.01883434266433,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100,
       "hard": 67.9144385026738,
       "judge": 76.26895187870798,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 28.205659075224297,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 30.434782608695656,
       "judge": 0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 57.66440539777861,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 66.90386806087209,
       "hard": 59.08254746680842,
       "judge": 54.11814783561007,
       "standard": 55.527776271425026
      }
     },
     "sealed|choice": {
      "cc": 68.42011936132744,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 83.10564188132234,
       "hard": 59.93184566340551,
       "judge": 68.95134696362436,
       "standard": 77.25706409372847
      }
     },
     "sealed|noul": {
      "cc": 20.840080971659916,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 64.1025641025641,
       "hard": 0,
       "judge": 30.555555555555557,
       "standard": 26.315789473684205
      }
     },
     "sealed|score": {
      "cc": 46.145794638077234,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 59.269920865495806,
       "hard": 31.19109112767333,
       "judge": 57.63492575992532,
       "standard": 52.25944186240362
      }
     }
    }
   },
   "jevbench_score": 68.7254259320887,
   "key": "blink-v0-3-26b-a4b-nvfp4",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (card tag; no LICENSE file in the repo); base google/gemma-4-26B-A4B-it Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.7",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "PixilabAI/Blink-v0.3-26B-A4B-NVFP4@08caad20c1574fb76341ca809ff38fdb081f7d4a",
   "noul_decisive": {
    "acc_among_decisive": 0.9467213114754098,
    "decisive": 244,
    "decisive_rate": 0.6703296703296703,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.67498392881862,
     "cost": 49.62845783533821,
     "intelligence": 49.21581563112197,
     "speed": 91.48151024194223
    },
    "board_release": "v1.6.43",
    "capability": 69.9453997799703,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 28,
    "capability_rank": 48,
    "composite_ci95": {
     "A": [
      45.23103654293889,
      64.78227929319183
     ],
     "B": [
      41.83419561295226,
      62.07842151813806
     ],
     "C": [
      31.410442043707555,
      52.48195783546016
     ]
    },
    "jevbench_score": 61.16059214787521,
    "measured_in": "v1.6.7",
    "method": "v1.6",
    "open_board_rank": 33,
    "rank": 57,
    "ranks": {
     "A": 57,
     "B": 57,
     "C": 62,
     "capability": 80
    },
    "scores": {
     "A": 61.16059214787521,
     "B": 57.67804533002703,
     "C": 42.47263343602446
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.252750841062516,
     "p50_s_raw": 0.05137542053125799,
     "p95_s_adjusted": 0.2812913428759202,
     "p95_s_raw": 0.06564567143796012
    },
    "views": {
     "Balanced 33:33:33": 55.71250293531964,
     "Emphasis on Accuracy 60:20:20": 51.855982453841044,
     "Emphasis on Cost 20:20:60": 52.04705676153498,
     "Emphasis on Speed 20:60:20": 65.14523654261257,
     "Intelligence only": 46.97811680206471
    }
   },
   "rank": 88,
   "ranked": true,
   "ranks": {
    "A": 88,
    "B": 101,
    "C": 106,
    "capability": 102
   },
   "repo": "https://huggingface.co/PixilabAI/Blink-v0.3-26B-A4B-NVFP4",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.7254259320887,
    "B": 63.15208401090139,
    "C": 47.72599023061716
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.25477972798980775,
    "p50_s_raw": 0.05238986399490386,
    "p95_s_adjusted": 0.30209081431385126,
    "p95_s_raw": 0.07604540715692563
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8620689655172413,
       "cc": 78.57219756866152,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 66.66666666666667,
        "score": 77.38325937265121
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 78.57219756866152
      },
      "calibration_gen": {
       "accuracy": 0.6607142857142857,
       "cc": 44.41501762938165,
       "cc_by_type": {
        "choice": 69.72049689440995,
        "noul": 15.384615384615374,
        "score": 48.139940609119634
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 44.41501762938165
      },
      "composite_gen": {
       "accuracy": 0.7755102040816326,
       "cc": 61.661977649287216,
       "cc_by_type": {
        "choice": 81.48148148148148,
        "noul": 28.57142857142858,
        "score": 74.93302289495159
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 61.661977649287216
      },
      "fairness": {
       "accuracy": 1,
       "cc": 98.74247115687324,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.22741347061967
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 98.74247115687324
      },
      "jaggedness": {
       "accuracy": 0.5243243243243243,
       "cc": 12.830160086648034,
       "cc_by_type": {
        "choice": 55.53496989346919,
        "noul": -58.97435897435898,
        "score": 41.92986934083389
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 12.830160086648034
      },
      "llm_router": {
       "accuracy": 0.7992424242424242,
       "cc": 72.61663013184426,
       "cc_by_type": {
        "choice": 65.62935438922435,
        "noul": 75.34246575342465,
        "score": 76.87807025288377
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 72.61663013184426
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4838709677419355,
       "cc": 21.95983819644401,
       "cc_by_type": {
        "choice": 35.38461538461538,
        "noul": 0,
        "score": 30.494899204716653
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 21.95983819644401
      },
      "multilingual": {
       "accuracy": 0.6984126984126984,
       "cc": 49.44236254308296,
       "cc_by_type": {
        "choice": 72.2346618898343,
        "noul": 26.436781609195403,
        "score": 49.655644130219166
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 49.44236254308296
      },
      "negation_pair_gen": {
       "accuracy": 0.7321428571428571,
       "cc": 43.01413459029695,
       "cc_by_type": {
        "choice": 90.47619047619048,
        "noul": -9.999999999999998,
        "score": 48.56621329470037
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 43.01413459029695
      },
      "option_order": {
       "accuracy": 0.9649122807017544,
       "cc": 95.5604883462819,
       "cc_by_type": {
        "choice": 95.5604883462819
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 95.5604883462819
      },
      "perturb_pair": {
       "accuracy": 0.9,
       "cc": 84.88006962790548,
       "cc_by_type": {
        "choice": 84.27260812581913,
        "noul": 85.71428571428572,
        "score": 84.6533150436116
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 84.88006962790548
      },
      "usecase": {
       "accuracy": 0.6286919831223629,
       "cc": 38.36326470609512,
       "cc_by_type": {
        "choice": 67.6238196184236,
        "noul": 19.298245614035082,
        "score": 28.167728885826694
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 38.36326470609512
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5454545454545454,
       "cc": 4.937825248418551,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": 0,
        "score": -5.186524254744351
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 4.937825248418551
      },
      "cs": {
       "accuracy": 0.75,
       "cc": 53.954802259887,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 53.954802259887
      },
      "da": {
       "accuracy": 0.7916666666666666,
       "cc": 54.43977410930783,
       "cc_by_type": {
        "choice": 64.49704142011835,
        "noul": 19.999999999999996,
        "score": 78.82228090780514
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 54.43977410930783
      },
      "de": {
       "accuracy": 0.8,
       "cc": 70.03622189564423,
       "cc_by_type": {
        "choice": 83.70549185274592,
        "noul": 77.77777777777777,
        "score": 48.625396056408995
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 70.03622189564423
      },
      "el": {
       "accuracy": 0.4444444444444444,
       "cc": -4.511613763017206,
       "cc_by_type": {
        "choice": 51.80722891566265,
        "noul": -60,
        "score": -5.3420702047142665
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.6979445933869526,
       "cc": 46.31123906153604,
       "cc_by_type": {
        "choice": 70.16264722378011,
        "noul": 19.548872180451138,
        "score": 49.222197780376874
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 46.31123906153604
      },
      "es": {
       "accuracy": 0.6785714285714286,
       "cc": 54.13125875630315,
       "cc_by_type": {
        "choice": 47.63636363636363,
        "noul": 63.63636363636365,
        "score": 51.12104899618216
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 54.13125875630315
      },
      "fi": {
       "accuracy": 0.3333333333333333,
       "cc": -8.920187793427232,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.7333333333333333,
       "cc": 39.085828039995185,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -50,
        "score": 67.25748411998555
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 39.085828039995185
      },
      "hi": {
       "accuracy": 0.5333333333333333,
       "cc": 38.75519442187261,
       "cc_by_type": {
        "choice": 50.31055900621117,
        "noul": 19.999999999999996,
        "score": 45.95502425940667
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 38.75519442187261
      },
      "id": {
       "accuracy": 0.6666666666666666,
       "cc": 33.87096774193548,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 33.87096774193548
      },
      "it": {
       "accuracy": 0.8823529411764706,
       "cc": 76.94126084433556,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 30.82378253300666
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 76.94126084433556
      },
      "ja": {
       "accuracy": 0.53125,
       "cc": 25.083725989475457,
       "cc_by_type": {
        "choice": 59.34959349593497,
        "noul": -14.28571428571429,
        "score": 30.187298758205692
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 25.083725989475457
      },
      "ko": {
       "accuracy": 0.7272727272727273,
       "cc": 74.31438742885162,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 22.943162286554863
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 74.31438742885162
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.95012548973524,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.85037646920576
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 99.95012548973524
      },
      "nl": {
       "accuracy": 0.6923076923076923,
       "cc": 21.532452114485192,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": -68.73597698987774
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 21.532452114485192
      },
      "no": {
       "accuracy": 0.5714285714285714,
       "cc": 33.87978142076503,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 33.87978142076503
      },
      "pl": {
       "accuracy": 0.65,
       "cc": 44.580498742442956,
       "cc_by_type": {
        "choice": 92.12942719720158,
        "noul": 0,
        "score": 41.612069030127294
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 44.580498742442956
      },
      "pt": {
       "accuracy": 0.9444444444444444,
       "cc": 90.22683698066082,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": 100,
        "score": 88.77096320328903
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 90.22683698066082
      },
      "sv": {
       "accuracy": 0.625,
       "cc": 36.356054299978204,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 53.51260734437906
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 36.356054299978204
      },
      "tr": {
       "accuracy": 0.8181818181818182,
       "cc": 60.31616533251732,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 100,
        "score": 7.264285471236143
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 60.31616533251732
      },
      "uk": {
       "accuracy": 0.6363636363636364,
       "cc": 10.754585109986635,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -100,
        "score": 75.74201619952514
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 10.754585109986635
      },
      "zh": {
       "accuracy": 0.875,
       "cc": 62.19801371486293,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": -13.405958855411203
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 62.19801371486293
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7693333333333333,
       "cc": 70.11573674209708,
       "cc_by_type": {
        "choice": 70.11573674209708
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 70.11573674209708
      },
      "noul": {
       "accuracy": 0.616,
       "cc": 23.2,
       "cc_by_type": {
        "noul": 23.2
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 23.2
      },
      "score": {
       "accuracy": 0.6293333333333333,
       "cc": 48.580321460438455,
       "cc_by_type": {
        "score": 48.580321460438455
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 48.580321460438455
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.75,
       "cc": 43.75694683730911,
       "cc_by_type": {
        "choice": 70.34596375617794,
        "noul": 19.999999999999996,
        "score": 40.924876755749395
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 43.75694683730911
      },
      "code_linting": {
       "accuracy": 0.6666666666666666,
       "cc": 30.4794447005702,
       "cc_by_type": {
        "choice": 70.54009819967267,
        "noul": 19.999999999999996,
        "score": 0.8982359020379249
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 30.4794447005702
      },
      "customer_support": {
       "accuracy": 0.65,
       "cc": 47.88048103401948,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 25,
        "score": 50.89950761818749
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 47.88048103401948
      },
      "demand_forecasting": {
       "accuracy": 0.19230769230769232,
       "cc": -40.46675998594265,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -100,
        "score": -25.01473778915324
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5,
       "cc": 30.134531521647336,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -19.999999999999996,
        "score": 47.670675310283606
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 30.134531521647336
      },
      "feature_extraction": {
       "accuracy": 0.5862068965517241,
       "cc": 40.83135316808231,
       "cc_by_type": {
        "choice": 65.81196581196579,
        "noul": 19.999999999999996,
        "score": 36.68209369228115
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 40.83135316808231
      },
      "financial_crime": {
       "accuracy": 0.6153846153846154,
       "cc": 43.836506452390005,
       "cc_by_type": {
        "choice": 73.91304347826087,
        "noul": 25,
        "score": 32.59647587890912
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 43.836506452390005
      },
      "gaming": {
       "accuracy": 0.5517241379310345,
       "cc": 13.446554121042317,
       "cc_by_type": {
        "choice": 52.475247524752476,
        "noul": -14.28571428571429,
        "score": 2.150129124088762
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 13.446554121042317
      },
      "insurance_claims": {
       "accuracy": 0.5,
       "cc": 39.067485028605155,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": 19.999999999999996,
        "score": 76.56753445089484
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 39.067485028605155
      },
      "knowledge_graphs": {
       "accuracy": 0.5652173913043478,
       "cc": 30.3227658858092,
       "cc_by_type": {
        "choice": 54.198473282442755,
        "noul": 19.999999999999996,
        "score": 16.769824374984854
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 30.3227658858092
      },
      "lead_generation": {
       "accuracy": 0.48,
       "cc": 30.591986352709828,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 0,
        "score": 8.036121659755512
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 30.591986352709828
      },
      "legal_compliance": {
       "accuracy": 0.6153846153846154,
       "cc": 61.8756462123043,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": 18.41382388281453
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 61.8756462123043
      },
      "llm_guardrails": {
       "accuracy": 0.7931034482758621,
       "cc": 60.58510134359349,
       "cc_by_type": {
        "choice": 81.65137614678899,
        "noul": 25,
        "score": 75.10392788399149
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 60.58510134359349
      },
      "model_routing": {
       "accuracy": 0.86,
       "cc": 72.32255475621157,
       "cc_by_type": {
        "choice": 88.74296435272043,
        "noul": 72.72727272727273,
        "score": 55.497427188641524
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 72.32255475621157
      },
      "moderation_trust_safety": {
       "accuracy": 0.875,
       "cc": 69.1293608581969,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 19.999999999999996,
        "score": 87.38808257459071
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 69.1293608581969
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 38.80908171961838,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 0,
        "score": 46.577998927699404
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 38.80908171961838
      },
      "risk_assessment": {
       "accuracy": 0.38461538461538464,
       "cc": -15.99553483012555,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": -100,
        "score": 8.61716909452901
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.5625,
       "cc": 27.277877295181455,
       "cc_by_type": {
        "choice": 60.1593625498008,
        "noul": -14.28571428571429,
        "score": 35.959983621457866
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 27.277877295181455
      },
      "search_retrieval": {
       "accuracy": 0.9090909090909091,
       "cc": 84.20501734874512,
       "cc_by_type": {
        "choice": 85.68019093078759,
        "noul": 100,
        "score": 66.93486111544775
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 84.20501734874512
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.77397078635877,
      90.58056567199262
     ],
     "I": [
      43.35399336962467,
      54.02584879853301
     ],
     "capability": [
      65.11784736173853,
      71.70425623732149
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.67498392881862,
     "capability": 69.9453997799703,
     "intelligence": 49.21581563112197
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7693333333333333,
      "calibration": 90.35563030686254,
      "cc": 70.11573674209708,
      "cc_by_type": {
       "choice": 70.11573674209708
      },
      "cc_tiered": 71.21947685199589,
      "cc_tiered_open": 74.01883434266433,
      "cc_tiered_sealed": 68.42011936132744,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.11573674209708
     },
     "noul": {
      "accuracy": 0.616,
      "calibration": 90.17262756314747,
      "cc": 23.2,
      "cc_by_type": {
       "noul": 23.2
      },
      "cc_tiered": 24.522870023442106,
      "cc_tiered_open": 28.205659075224297,
      "cc_tiered_sealed": 20.840080971659916,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 23.2
     },
     "score": {
      "accuracy": 0.6293333333333333,
      "calibration": 91.49669391644588,
      "cc": 48.580321460438455,
      "cc_by_type": {
       "score": 48.580321460438455
      },
      "cc_tiered": 51.90510001792792,
      "cc_tiered_open": 57.66440539777861,
      "cc_tiered_sealed": 46.145794638077234,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 48.580321460438455
     }
    },
    "raw_on_A": {
     "calibration": 87.63270878627716,
     "capability": 67.620060964045,
     "intelligence": 47.60741314181283
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.07521120298289,
    "Emphasis on Accuracy 60:20:20": 56.33000817130341,
    "Emphasis on Cost 20:20:60": 65.90446678199271,
    "Emphasis on Speed 20:60:20": 71.97365455632244,
    "Intelligence only": 47.68415070868776
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "sanghwa-na (Hugging Face; code foxl-ai/bobcat on GitHub)",
   "axes": {
    "calibration": 87.62628477134197,
    "cost": 55.19130539235455,
    "intelligence": 68.0935397642001,
    "speed": 89.29862693377845
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04389435292018364,
      "ece_score": 91.22112941596328,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 90.0048476904171,
      "score_before_time_limit": 91.22112941596328,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.07999999999999996,
      "brier": 0.08188541684317341,
      "ece": 0.0771803607955985,
      "n": 375,
      "score": 84.11292022572896,
      "score_before_time_limit": 84.56392784088031,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9413333333333334,
      "interval90_width": 0.29177777777777764,
      "n": 375,
      "nrps": 0.06991851787155044,
      "score": 88.7610863978799,
      "score_before_time_limit": 89.47690161076602,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.07027172495656454
     }
    },
    "score": 87.62628477134197
   },
   "capability": 77.85991226777104,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 773,587 input + 0 output tokens over 1,485 decisions = 520.934006734 tokens/decision. GPU-time cost USD 0.0311618443215105/1,000; old list-price cost USD 0.0468840606060606/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0311618443215105/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.11860985999999998
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.031161844321510472
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bobcat 1.3 (Gemma 4 31B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h. The authors' flash_server (vLLM engine, online FP8, 127.0.0.1:8000) and the 18 hash-pinned author files run in ONE docker container with network none, served at --max-model-len 98368 with --max-num-seqs 64; the driver sent requests one at a time over loopback on the VM. Network cut proven from inside the container before any item hash or model load (loopback only, no default route, DNS and TCP fail); read-only code, read-only model and input, empty credential variables (names-only proof), HF offline flags. Measured 23 Oct 2026 (VM 06:47-07:12Z; measurement 855 s, total 1,475 s; scored run 673 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, re-checked inside the container). The input was shredded from the node and its absence verified, and the VM was removed by exact id afterwards (teardown confirmed 07:12Z).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 71.27885942153964,
    "I_sealed": 64.90822010686055,
    "base": 68.0935397642001,
    "excess": -3.4688874556766294,
    "gap": 6.370639314679082,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 74.33918859210414,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 67.9144385026738,
       "judge": 72.31377719182598,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 81.04347826086956,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 60.00000000000001,
       "standard": 100
      }
     },
     "open|score": {
      "cc": 58.45391141164524,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.90713302535477,
       "hard": 68.44215068172673,
       "judge": 54.57383133604144,
       "standard": 39.070942178033185
      }
     },
     "sealed|choice": {
      "cc": 78.21398393688209,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 76.58147334335607,
       "judge": 74.43682664054849,
       "standard": 81.01666315123391
      }
     },
     "sealed|noul": {
      "cc": 66.22701539325317,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 61.81818181818181,
       "judge": 60.00000000000001,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 50.28366099044642,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 67.49288080741778,
       "hard": 49.582586651545235,
       "judge": 45.11116758489915,
       "standard": 50.839939868083995
      }
     }
    }
   },
   "jevbench_score": 72.1826764470016,
   "key": "bobcat-1.3",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 (model card and Hub metadata; base google/gemma-4-31B-it: apache-2.0 in its card metadata with a link to the Gemma 4 licence), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "sanghwa-na/bobcat-1.3@4f25c70ece312ec4e74c47f1c5264cb2aeea956a",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.936046511627907,
    "decisive": 344,
    "decisive_rate": 0.9222520107238605,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 54,
   "ranked": true,
   "ranks": {
    "A": 54,
    "B": 42,
    "C": 42,
    "capability": 46
   },
   "repo": "https://huggingface.co/sanghwa-na/bobcat-1.3",
   "scores": {
    "A": 72.1826764470016,
    "B": 71.32602581724223,
    "C": 72.1826764470016
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2878270249999787,
    "p50_s_raw": 0.06891351249998934,
    "p95_s_adjusted": 0.4083247809998397,
    "p95_s_raw": 0.12916239049991987
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run unmodified: foxl-ai/bobcat at 538169365291d7c46d2c116f8ee09ad9f4b3ac5e (5 commits past the previously reviewed 43387a09, +6,032/-45 lines over 34 files, none in the serving path: the 18 serving modules are byte-identical at both commits). Only those 18 files (src/bobcat/*.py, manifest sha256 71dad08892d1e489b312bcd3fffa8a3bcef529181ad1db16deb9c04610d4ff83) were shipped to the VM, read-only at /code/src, hash-verified; every other module of the repo raises ImportError; no git clone, no pip install of the author's package. Command (the 1.3 card's Quickstart plus --identifiers): python3 -m bobcat.flash_server --engine vllm --model /models/bobcat-1.3 --compiler-model /models/bobcat-1.3/compiler --quantization fp8 --temperature choice=1.007793,noul=0.445449,score=1.023435 --name bobcat-1.3 --release-date 2026-10-11 --max-num-seqs 64 --max-model-len 98368 --schedule all --engine-arg max_num_batched_tokens=16384 --engine-arg attention_backend=TRITON_ATTN --identifiers /models/bobcat-1.3/bobcat-identifiers.json --host 127.0.0.1 --port 8000 --local (online FP8 of the BF16 weights, the author's recommended path; no kv_cache_dtype override). Image: official vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 plus ONE hash-pinned wheel (scipy 1.15.3), built offline on the VM (image sha256:42b2ad6131642a63abe73bb9c7fa0690ae47356e1be2bb39988971ef5b00d742); vllm 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, tokenizers 0.23.2. The submitted code was reviewed and cleared for this run because the unchanged serving modules contain dormant hooks (DISCLOSED CODE PATTERNS, none reachable in our setup: a Hub downloader and a model-provided-code exec_module fallback in student_readout.py:21,57-63,334-346 (the model repo has no encoding/ directory and ships a chat template, the container has no network); an env read of BOBCAT_EDGE_SECRET at flash_server.py:87 (empty, as --local allows); boto3 at checkpoints.py:102, hf_hub_download at corpus.py:69, httpx at glm_readout.py:191 and torch.load(weights_only=True) at serve.py:33, all training or command-line paths not reachable from flash_server). The source changes and runner were reviewed before the run, findings were fixed and the fixes rechecked. Weights: sanghwa-na/bobcat-1.3 at 4f25c70ece312ec4e74c47f1c5264cb2aeea956a (Apache-2.0, public, not gated, 22 files, BF16 62,546,338,248 weight bytes; the weight average of three merged rank-16 LoRA arms on google/gemma-4-31B-it), fetched by us from the commit-pinned Hub URLs and sha256-verified file by file (MODEL_MANIFEST.json sha256 882e017a6873c6e312277f39db1c91795f440f79a7dfa00e08258a5b31b694f2). FIRST MEASUREMENT (found by our own intake; Author submission, a submission by the authors; our triage treats it as superseding their Bobcat 1.2 submission author submission; the earlier row bobcat-flash-1.2 is a different model). AUTHOR'S OWN SERVER, on our hardware: the row measures sanghwa-na/bobcat-1.3 served by foxl-ai/bobcat as the card's Quickstart documents, in online FP8, on one RTX PRO 6000 (not the author's B200). 10 s RULE: 15 decisions took longer than 10 s (35-38 s each, max 38.5 s) and are scored wrong under the rule in the v1.7 share view (the frozen v1.6 and v1.7-off views do not apply the 10 s rule); they are exactly the 15 standard long positions (76,000-82,000 tokens), consistent with a cold prefill of the whole state (in the smoke a 78.5k-token request took 34.8 s cold and 0.12 s warm); no sealed item was repeated warm. The long items are answered in full (never truncated, never refused). The other 1,485 decisions were fast (p50 0.069 s, p95 0.129 s; the 99th percentile over all 1,500 is 35.1 s only because of these 15). DETERMINISM: first 50 positions run twice, 48/50 bit-identical, 0 argmax flips, max probability difference 0.0012 (online FP8 server). OPEN PARITY RISK: the 1.3 release manifest binds a student_readout.py (sha256 e5597653...) that was never published; the public file (f8d3ceb8e1f33509..., the only version ever pushed, also bound by the 1.2 manifests) is what the card's Quickstart installs and what we ran, so parity with the author's own 1.3 evaluation is unproven. AUTHOR EXPOSURE (card, self-reported, not verified): the JevBench public 231 was evaluated three times by the authors (unofficial Capability 77.2); the Decision Index public half was used for candidate selection; the Choice and Score temperatures were fitted on LocalLLaMA/typed-decisions even test ids and the Noul temperature on a held-out JevBench-format split. The authors' delta audit code drops training rows that overlap the public 231; we did not audit it, ran no overlap check and cannot rule out other overlap. The fitted temperatures (choice 1.007793, noul 0.445449, score 1.023435) are part of the served configuration. The card's latency numbers (61.4 ms on RTX PRO 6000 for Bobcat 1.2, 28.6 ms on B200 for 1.3) are the author's and do not transfer. TOKEN COUNTING: our counter was validated by one native datum from the earlier bobcat-flash-1.2 run (105) and then by the sealed item-by-item comparison, which matched exactly; the image used tokenizers 0.23.2, the offline counter 0.22.2. No outside request for this model; priced as an estimate from the base model, not from any tariff of the authors.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 74.23969124852903,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 39.38574041225378
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 74.23969124852903
      },
      "calibration_gen": {
       "accuracy": 0.6336633663366337,
       "cc": 52.81534073389674,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 52,
        "score": 54.1660646195057
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 52.81534073389674
      },
      "composite_gen": {
       "accuracy": 0.9306930693069307,
       "cc": 85.33266164130478,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 92.59259259259258,
        "score": 73.16148989229738
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.33266164130478
      },
      "fairness": {
       "accuracy": 0.9583333333333334,
       "cc": 92.62055492372407,
       "cc_by_type": {
        "choice": 100,
        "noul": 86.66666666666667,
        "score": 91.19499810450556
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 92.62055492372407
      },
      "jaggedness": {
       "accuracy": 0.6055555555555555,
       "cc": 43.755252658109974,
       "cc_by_type": {
        "choice": 55.24542829643889,
        "noul": 51.35135135135136,
        "score": 24.66897832653968
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 43.755252658109974
      },
      "llm_router": {
       "accuracy": 0.9137380191693291,
       "cc": 83.06639153155028,
       "cc_by_type": {
        "choice": 92.86783481553225,
        "noul": 93.81443298969072,
        "score": 62.51690678942784
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.06639153155028
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 52.90360670916882,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 50,
        "score": 33.45308816874354
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 52.90360670916882
      },
      "multilingual": {
       "accuracy": 0.7818181818181819,
       "cc": 63.96786213240862,
       "cc_by_type": {
        "choice": 74.5679817411151,
        "noul": 55.55555555555556,
        "score": 61.7800491005552
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 63.96786213240862
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 78.96146717862894,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 80,
        "score": 63.55106820255351
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 78.96146717862894
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 84.6306841840915,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 65.90406456428653
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 84.6306841840915
      },
      "usecase": {
       "accuracy": 0.7136363636363636,
       "cc": 59.83498229574659,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 58.33333333333333,
        "score": 48.13239786763192
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 59.83498229574659
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 37.142210792909026,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": -0.4781295260348317
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 37.142210792909026
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 58.609331259061015,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 63.23198763278983
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 58.609331259061015
      },
      "de": {
       "accuracy": 0.875,
       "cc": 87.29697354232822,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 85.78472593671917
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.29697354232822
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.37645472611915,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.31558391764983
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.37645472611915
      },
      "en": {
       "accuracy": 0.8001737619461338,
       "cc": 68.78566887480646,
       "cc_by_type": {
        "choice": 78.91165594757203,
        "noul": 74.82014388489209,
        "score": 52.62520679195526
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 68.78566887480646
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 71.02044939362587,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 22.849276402737296
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.02044939362587
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.32391955062916,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 98.42531056445578
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.32391955062916
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 78.71231633263812,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 78.48989017438495
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 78.71231633263812
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 74.23237617511566,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 75,
        "score": 47.697128525346976
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 74.23237617511566
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 62.194486664910634,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 0.6130780773507594
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 62.194486664910634
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 45.72426237348003,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 60.00000000000001,
        "score": 5.766909439741108
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 45.72426237348003
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 47.59296668875374,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 87.22334451070569
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 47.59296668875374
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.10819870917948,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 97.9912627942051
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.10819870917948
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 24.413351253396552,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 32.54741306754896
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 24.413351253396552
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 63.418062521802085,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 57.871623028800265
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 63.418062521802085
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 69.53317456601008,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 51.027337487069715
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 69.53317456601008
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 47.301971952842074,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 32.96974564576027
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 47.301971952842074
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 76.0081598974826,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 49.763610127230415
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 76.0081598974826
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 76.83303262949057,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 62.757162404600784
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 76.83303262949057
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 71.27600077139918,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 13.82800231419752
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 71.27600077139918
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.832,
       "cc": 78.30038844983639,
       "cc_by_type": {
        "choice": 78.30038844983639
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 78.30038844983639
      },
      "noul": {
       "accuracy": 0.8586666666666667,
       "cc": 71.73333333333333,
       "cc_by_type": {
        "noul": 71.73333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 71.73333333333333
      },
      "score": {
       "accuracy": 0.6693333333333333,
       "cc": 53.323643295776094,
       "cc_by_type": {
        "score": 53.323643295776094
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 53.323643295776094
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6818181818181818,
       "cc": 48.21772340389142,
       "cc_by_type": {
        "choice": 84,
        "noul": 25,
        "score": 35.653170211674244
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 48.21772340389142
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 65.04142526820455,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 17.346498026835878
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 65.04142526820455
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 62.52068687198607,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100,
        "score": 3.822223217584231
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 62.52068687198607
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": 11.507558082242374,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": -5.243407624617902
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 11.507558082242374
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 56.64084608163887,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 65.57471215796006
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 56.64084608163887
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 50.05850231278627,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 66.66666666666667,
        "score": 46.401922032698415
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 50.05850231278627
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 41.44357485132011,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 39.456774974128386
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 41.44357485132011
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 28.0719633890198,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -50,
        "score": 65.574775184481
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 28.0719633890198
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 49.2441230583897,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 100,
        "score": 10.232369175169087
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 49.2441230583897
      },
      "knowledge_graphs": {
       "accuracy": 0.6818181818181818,
       "cc": 65.63025255294463,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100,
        "score": 35.43465487510799
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 65.63025255294463
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 49.11013862345109,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 3.4027672915419194
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 49.11013862345109
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 59.40395296198992,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 42.727987918227825
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 59.40395296198992
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 75.06726419420765,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 65.20179258262291
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 75.06726419420765
      },
      "model_routing": {
       "accuracy": 0.8986175115207373,
       "cc": 81.38800385735159,
       "cc_by_type": {
        "choice": 92.98701298701299,
        "noul": 100,
        "score": 51.17699858504179
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.38800385735159
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 23.34421860436829,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 81.5489705923755
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 23.34421860436829
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.85494744780317,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 64.65769285408733
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.85494744780317
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 48.05391633532412,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50,
        "score": 23.718399252277777
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 48.05391633532412
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 36.046095497196575,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 51.61654736115495
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 36.046095497196575
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.66123219687104,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.98369659061314
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.66123219687104
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.16269651951559,
      88.93073363136115
     ],
     "I": [
      63.4644225486904,
      72.14683687162247
     ],
     "capability": [
      74.05948716208472,
      79.9460707309049
     ],
     "composite": null,
     "seed": 23
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.62628477134197,
     "capability": 77.85991226777104,
     "intelligence": 68.0935397642001
    },
    "per_type": {
     "choice": {
      "accuracy": 0.832,
      "calibration": 90.0048476904171,
      "cc": 78.30038844983639,
      "cc_by_type": {
       "choice": 78.30038844983639
      },
      "cc_tiered": 76.27658626449312,
      "cc_tiered_open": 74.33918859210414,
      "cc_tiered_sealed": 78.21398393688209,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 78.30038844983639
     },
     "noul": {
      "accuracy": 0.8586666666666667,
      "calibration": 84.11292022572896,
      "cc": 71.73333333333333,
      "cc_by_type": {
       "noul": 71.73333333333333
      },
      "cc_tiered": 73.63524682706137,
      "cc_tiered_open": 81.04347826086956,
      "cc_tiered_sealed": 66.22701539325317,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 71.73333333333333
     },
     "score": {
      "accuracy": 0.6693333333333333,
      "calibration": 88.7610863978799,
      "cc": 53.323643295776094,
      "cc_by_type": {
       "score": 53.323643295776094
      },
      "cc_tiered": 54.36878620104583,
      "cc_tiered_open": 58.45391141164524,
      "cc_tiered_sealed": 50.28366099044642,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.323643295776094
     }
    },
    "raw_on_A": {
     "calibration": 85.92539384072604,
     "capability": 77.22561866144176,
     "intelligence": 68.52584348215746
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.17739073530349,
    "Emphasis on Accuracy 60:20:20": 68.14382557790856,
    "Emphasis on Cost 20:20:60": 62.312711945602096,
    "Emphasis on Speed 20:60:20": 75.30163768160718,
    "Intelligence only": 68.0935397642001
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Foxl AI / The Bobcat Authors (admin@foxl.ai)",
   "axes": {
    "calibration": 80.53028652577869,
    "cost": 71.06558799286785,
    "intelligence": 65.56939608070681,
    "speed": 90.60003447236298
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.114937134190201,
      "ece_score": 77.01257316195979,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 76.19110571489888,
      "score_before_time_limit": 77.01257316195979,
      "time_limit_failed_share": 0.010666666666666666
     },
     "noul": {
      "abstention_rate": 0.10666666666666669,
      "brier": 0.08802311851821586,
      "ece": 0.07573463232351335,
      "n": 375,
      "score": 84.40052380977575,
      "score_before_time_limit": 84.85307353529733,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9066666666666666,
      "interval90_width": 0.1808888888888889,
      "n": 375,
      "nrps": 0.06948590052038994,
      "score": 80.99923005266146,
      "score_before_time_limit": 81.65244964986034,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.14873255324120163
     }
    },
    "score": 80.53028652577869
   },
   "capability": 73.04984130324274,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 780,331 input + 0 output tokens over 1,477 decisions = 528.321597833 tokens/decision. GPU-time cost USD 0.00921468301821138/1,000; old list-price cost USD 0.0475489438050102/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00921468301821138/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.15782652
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009214683018211383
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 13,
    "n_over_limit_answered_ok": 13,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bobcat Flash 1.2 (Gemma-4-26B-A4B-it + two merged rank-64 LoRA adapters, typed-decision readout at the first answer position, FP8, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights sanghwa-na/bobcat-flash-1.2@58d15456937ce58625b13a5e3d982846554a3aef, Apache-2.0, 51.68 GB BF16 - a merge of two rank-64 LoRA adapters on google/gemma-4-26B-A4B-it@4d7ae4984b7db7de8f8457170b3f1a419ee76d52 (0.6*A + 0.4*B, B being the Flash 1.1 adapter), file-by-file verified against the Hub's own manifest before the engine was started. Code github.com/foxl-ai/bobcat@43387a09bbc0310edcf57e2b21fa59ea81ac4166, Apache-2.0. NO MAPPING DECISION OF OURS EXISTS FOR THIS ROW, which is why it is worth saying so explicitly. The author implements TypeSafe's published System One contract, and their own src/bobcat/protocol.py parse_request accepts the JevBench canonical record EXACTLY as the UNCHANGED jevbench.adapters.typesafe already emits it: choice -> a dict of 1-255 named criteria, so the item's own labels are the keys and its own criteria[label] strings travel verbatim as the descriptions (their protocol renders {name, description} per option); score -> the item's ordered array of levels, which their own parse_request labels '0'..'n-1' - the same keys JevBench uses - with the item's own level text as each description, so the ordinal order is carried; noul -> criteria restricted to {true,false}, and the item's OWN true/false description strings are delivered (unlike run 56's APUS rows, whose runtime required canonical criteria there), with the answer returned as noul = P(yes). Their answer() refuses to reply unless the probabilities sum to one. Nothing was rewritten, no option was dropped, no item type was recorded as a miss, and no shim of ours sits anywhere in the path. STACK: the author's own published install line, verbatim from their model card's Quickstart - a uv-managed Python 3.12 (their note: it ships the headers Triton compiles against) and vllm==0.30.0 LETTING IT RESOLVE ITS OWN STACK, which gave torch 2.13.0+cu130, transformers 5.19.0, tokenizers 0.23.2. Served by THEIR OWN SERVER, bobcat.flash_server, with THEIR OWN published command verbatim: --engine vllm --quantization fp8 (their online dynamic FP8 of these BF16 weights, the served configuration) --temperature choice=0.594604,noul=0.353553,score=0.529732 --max-num-seqs 64 --max-model-len 98368 --schedule all --engine-arg max_num_batched_tokens=16384 --engine-arg attention_backend=TRITON_ATTN, with VLLM_USE_FLASHINFER_SAMPLER=0 as their command sets it. --local only because the server is on loopback on our own pod. One H100 PCIe 80 GB (Lium) - the second of the two cards the author measured. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. No file of theirs was edited and no prompt of ours exists. CALIBRATION IS THEIR PUBLISHED DEFAULT AND WAS NOT CHOSEN BY US. Their card ships two calibrations: the per-type triple used here, which is the Quickstart's - i.e. the published serve command of this release - and a single product temperature 0.9232 fitted by log-loss on their six product tasks. The triple was bound in PREREG-R57 s3.3 before any item was sent and the alternative was deliberately NOT probed on our items, because the author has ALREADY PUBLISHED both configurations' numbers on JevBench's public 231 (per-type Capability 68.2 = 57.5/78.9; product 68.3 = 54.3/82.3), so there was nothing to learn from our pool and choosing between them from our own measurement would have been exactly the mistake run 56's probe was designed to avoid. The author states the triple was fitted on a held-out split of 8,906 JevBench-FORMAT decisions (generated and public) for the highest JevBench-style Capability, and separately that no JevBench item was used for training, calibration, selection or prompt design, with generated rows audited against JevBench's public items at 0 exact and 0 8-gram overlaps. THE 23 LONG ITEMS: the server's limit is --max-model-len 98368 (their compiler gets max_model_len - 64) against a pool whose longest item is 128,111 BYTES of canonical record, and their RequestLimitError answers 422 with 'Inputs are never truncated' rather than truncating. This row therefore reads every item in full. Serial, one decision per request, no retry, no batching, offline, credential-free. add-requests run 57, benchmarkheaven.com/submit ref 7A714784 (bh-submit e9d576d7-e1f1-49d8-8151-05cdbbdd7ed4)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100 PCIe",
   "intelligence": {
    "I_open": 65.86976952005253,
    "I_sealed": 65.26902264136108,
    "base": 65.56939608070681,
    "excess": -1.9726407637467909,
    "gap": 0.6007468786914529,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 74.45043236055656,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 66.02705253224286,
       "judge": 80.22412656558997,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 66.6425120772947,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 65.21739130434783,
       "judge": 50,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 56.516364122306314,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.947350572836605,
       "hard": 60.92245146041372,
       "judge": 57.42924259191816,
       "standard": 43.1193785164086
      }
     },
     "sealed|choice": {
      "cc": 75.92775512086644,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 78.71554208853745,
       "hard": 67.42081447963801,
       "judge": 81.3780260707635,
       "standard": 83.37233649464221
      }
     },
     "sealed|noul": {
      "cc": 59.790998006434116,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 37.86407766990292,
       "judge": 71.05263157894737,
       "standard": 74.64788732394365
      }
     },
     "sealed|score": {
      "cc": 60.088314796782676,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 85.14251727623912,
       "hard": 53.13372651343875,
       "judge": 54.946113599979476,
       "standard": 69.18369191894709
      }
     }
    }
   },
   "jevbench_score": 75.7895318580443,
   "key": "bobcat-flash-1.2",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "sanghwa-na/bobcat-flash-1.2@58d15456937ce58625b13a5e3d982846554a3aef",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.924924924924925,
    "decisive_rate": 0.8927613941018767,
    "supported": true,
    "valid": 373
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.17269878237248,
     "cost": 49.68577376542501,
     "intelligence": 65.81464576505633,
     "speed": 90.781726629608
    },
    "board_release": "v1.6.43",
    "capability": 73.4936722737144,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 14,
    "capability_rank": 30,
    "composite_ci95": {
     "A": [
      65.41706710923236,
      68.6674732651027
     ],
     "B": [
      64.1802436512738,
      68.73104866859511
     ],
     "C": [
      65.32359947723424,
      68.6674732651027
     ]
    },
    "jevbench_score": 67.34032787154446,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 23,
    "rank": 43,
    "ranks": {
     "A": 43,
     "B": 35,
     "C": 35,
     "capability": 54
    },
    "scores": {
     "A": 67.34032787154446,
     "B": 66.85676394909423,
     "C": 67.34032787154446
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2719114740379155,
     "p50_s_raw": 0.060955737018957734,
     "p95_s_adjusted": 0.3071848569670692,
     "p95_s_raw": 0.0785924284835346
    },
    "views": {
     "Balanced 33:33:33": 63.93311146268058,
     "Emphasis on Accuracy 60:20:20": 64.35172177912958,
     "Emphasis on Cost 20:20:60": 57.02049836458103,
     "Emphasis on Speed 20:60:20": 72.21838604704122,
     "Intelligence only": 64.99001759823823
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 11,
   "ranked": true,
   "ranks": {
    "A": 11,
    "B": 24,
    "C": 10,
    "capability": 76
   },
   "repo": "https://huggingface.co/sanghwa-na/bobcat-flash-1.2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 75.7895318580443,
    "B": 73.49833206659328,
    "C": 75.7895318580443
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2730665770126507,
    "p50_s_raw": 0.061533288506325334,
    "p95_s_adjusted": 0.3189539658022113,
    "p95_s_raw": 0.08447698290110565
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-26B-A4B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.59302008922947,
      83.34940918083149
     ],
     "I": [
      59.965147789446455,
      69.81159689545017
     ],
     "capability": [
      68.94776093285044,
      76.03279055626672
     ],
     "composite": null,
     "seed": 24
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 80.53028652577869,
     "capability": 73.04984130324274,
     "intelligence": 65.56939608070681
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8106666666666666,
      "calibration": 76.19110571489888,
      "cc": 75.78697081363734,
      "cc_by_type": {
       "choice": 75.78697081363734
      },
      "cc_tiered": 75.1890937407115,
      "cc_tiered_open": 74.45043236055656,
      "cc_tiered_sealed": 75.92775512086644,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.78697081363734
     },
     "noul": {
      "accuracy": 0.8213333333333334,
      "calibration": 84.40052380977575,
      "cc": 64.26666666666667,
      "cc_by_type": {
       "noul": 64.26666666666667
      },
      "cc_tiered": 63.2167550418644,
      "cc_tiered_open": 66.6425120772947,
      "cc_tiered_sealed": 59.790998006434116,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 64.26666666666667
     },
     "score": {
      "accuracy": 0.6933333333333334,
      "calibration": 80.99923005266146,
      "cc": 61.138841659786316,
      "cc_by_type": {
       "score": 61.138841659786316
      },
      "cc_tiered": 58.3023394595445,
      "cc_tiered_open": 56.516364122306314,
      "cc_tiered_sealed": 60.088314796782676,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 61.138841659786316
     }
    },
    "raw_on_A": {
     "calibration": 76.02127380259917,
     "capability": 72.29741756972824,
     "intelligence": 68.57356133685731
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.010666666666666713,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 74.33093070356979,
    "Emphasis on Accuracy 60:20:20": 70.55959602315767,
    "Emphasis on Cost 20:20:60": 72.98943462899726,
    "Emphasis on Speed 20:60:20": 80.08316384719262,
    "Intelligence only": 65.56939608070681
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "AgentBull (@0xDing)",
   "axes": {
    "calibration": 84.35778065081391,
    "cost": null,
    "intelligence": 25.21389259164919,
    "speed": 90.4627802720504
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07912071749672457,
      "ece_score": 84.17585650065509,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 84.17585650065509,
      "score_before_time_limit": 84.17585650065509,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.7426273458445041,
      "brier": 0.16309961620902136,
      "ece": 0.09024781687300962,
      "n": 373,
      "score": 81.95043662539807,
      "score_before_time_limit": 81.95043662539807,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9731182795698925,
      "interval90_width": 0.7293906810035842,
      "n": 372,
      "nrps": 0.1309150904455136,
      "score": 86.9470488263886,
      "score_before_time_limit": 86.9470488263886,
      "time_limit_failed_share": 0,
      "top_ece": 0.0650719665133573
     }
    },
    "score": 84.35778065081391
   },
   "capability": 54.78583662123155,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 dcfcecb0...2835, and every later re-freeze, which only add hosted-API launch-list keys) RAISES on the declared base google/t5gemma-2-4b-4b ('base model lacks an explicit market-listing decision'); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens as reported by the author's server (input 786,538, output_tokens 0).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bongard-mini (AgentBull, T5Gemma-2 4B-4B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own `bongard serve` with their temperatures.json, default token budgets (64,000 request / 32,768 sequence), bf16.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 29.129004250783858,
    "I_sealed": 21.29878093251452,
    "base": 25.21389259164919,
    "excess": -2.0093034520863746,
    "gap": 7.830223318269336,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 68.15046810038176,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 69.80182447310474,
       "judge": 64.40342781806197,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0,
       "hard": -56.52173913043479,
       "judge": -50,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 19.23654465196982,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 35.916731669865264,
       "hard": 9.296542606261049,
       "judge": 38.265686721521966,
       "standard": 2.2327421301114203
      }
     },
     "sealed|choice": {
      "cc": 47.46015197187779,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 61.88055908513342,
       "hard": 40.152654099687744,
       "judge": 47.11067580803136,
       "standard": 55.38915840539971
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -31.034482758620683,
       "hard": -60,
       "judge": -72.3076923076923,
       "standard": -58.2089552238806
      }
     },
     "sealed|score": {
      "cc": 16.436190825665776,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 26.462361570919747,
       "hard": 11.017848733353297,
       "judge": 15.93709798260039,
       "standard": 23.008428902261812
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "bongard-mini",
   "last_measured_on": "2026-10-16",
   "licence": "Gemma terms of use",
   "listing": "listed",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "AgentBull/bongard-mini@c81c5b883a5983669be84be98658eac940489613",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base google/t5gemma-2-4b-4b and raises rather than estimating; one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.9375,
    "decisive": 96,
    "decisive_rate": 0.257372654155496,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.35778065081391,
     "cost": null,
     "intelligence": 25.21389259164919,
     "speed": 90.58298788933249
    },
    "board_release": "v1.6.43",
    "capability": 54.78583662123155,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2890852989861742,
     "p50_s_raw": 0.06954264949308708,
     "p95_s_adjusted": 0.3024650537088746,
     "p95_s_raw": 0.07623252685443731
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/AgentBull/bongard-mini",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.28926610301132316,
    "p50_s_raw": 0.06963305150566157,
    "p95_s_adjusted": 0.31075950998929325,
    "p95_s_raw": 0.08037975499464664
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5,
       "cc": 0.4622404706696545,
       "cc_by_type": {
        "choice": 41.666666666666664,
        "noul": -50,
        "score": 9.7200547453423
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0.4622404706696545
      },
      "calibration_gen": {
       "accuracy": 0.21782178217821782,
       "cc": -35.922342791778796,
       "cc_by_type": {
        "choice": 10.92258748674444,
        "noul": -100,
        "score": -18.689615862080842
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.5841584158415841,
       "cc": 19.530539881920774,
       "cc_by_type": {
        "choice": 80.48780487804878,
        "noul": -40.74074074074075,
        "score": 18.8445555084543
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 19.530539881920774
      },
      "fairness": {
       "accuracy": 0.5625,
       "cc": 21.117700941050316,
       "cc_by_type": {
        "choice": 100,
        "noul": -60,
        "score": 23.35310282315095
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 21.117700941050316
      },
      "jaggedness": {
       "accuracy": 0.26666666666666666,
       "cc": -28.736348312955844,
       "cc_by_type": {
        "choice": 16.265640038498567,
        "noul": -94.5945945945946,
        "score": -7.88009038277151
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.5942492012779552,
       "cc": 29.786361842610262,
       "cc_by_type": {
        "choice": 61.96178568283877,
        "noul": -11.340206185567014,
        "score": 38.73750603055903
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 29.786361842610262
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -20.572694854487796,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100,
        "score": 12.50871956024795
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5272727272727272,
       "cc": 10.133806197798606,
       "cc_by_type": {
        "choice": 60.873818063254,
        "noul": -61.111111111111114,
        "score": 30.638711641252925
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 10.133806197798606
      },
      "negation_pair_gen": {
       "accuracy": 0.6046511627906976,
       "cc": 15.634962703773246,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": -53.333333333333336,
        "score": 16.90488811131974
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 15.634962703773246
      },
      "option_order": {
       "accuracy": 0.7291666666666666,
       "cc": 65.37949400798935,
       "cc_by_type": {
        "choice": 65.37949400798935
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 65.37949400798935
      },
      "perturb_pair": {
       "accuracy": 0.71875,
       "cc": 43.09664377241094,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 19.999999999999996,
        "score": 42.32296435026585
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 43.09664377241094
      },
      "usecase": {
       "accuracy": 0.41818181818181815,
       "cc": 0.5246135780320955,
       "cc_by_type": {
        "choice": 46.07843137254903,
        "noul": -70.83333333333333,
        "score": 26.32874269488058
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0.5246135780320955
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -17.34084978485645,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": -50,
        "score": -25.83207316409315
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 39.147237797438464,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 50.77504672564872
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 39.147237797438464
      },
      "de": {
       "accuracy": 0.65625,
       "cc": 43.40220333009313,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 25,
        "score": 45.02961883983691
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 43.40220333009313
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 19.633041242538255,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -100,
        "score": 75.6589002639276
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 19.633041242538255
      },
      "en": {
       "accuracy": 0.473501303214596,
       "cc": 3.7280108956536004,
       "cc_by_type": {
        "choice": 49.91518287548356,
        "noul": -53.23741007194245,
        "score": 14.506259883419691
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 3.7280108956536004
      },
      "es": {
       "accuracy": 0.7096774193548387,
       "cc": 34.3819807991804,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 0,
        "score": 12.933870619400889
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 34.3819807991804
      },
      "fi": {
       "accuracy": 0.25,
       "cc": 0.3453151180197314,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": 66.60971584586247
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0.3453151180197314
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 38.936757527017214,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 37.98674316928694
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 38.936757527017214
      },
      "hi": {
       "accuracy": 0.38095238095238093,
       "cc": -13.009001358504877,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100,
        "score": 11.252325533423912
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 43.06232479155725,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 62.5203077080051
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 43.06232479155725
      },
      "ja": {
       "accuracy": 0.4,
       "cc": -12.03076737823514,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": -80,
        "score": -27.498179815404388
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -9.868365203541524,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": -29.605095610624566
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.7857142857142857,
       "cc": 65.30590654320713,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 62.584386296288066
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 65.30590654320713
      },
      "nl": {
       "accuracy": 0.3,
       "cc": -19.67037363341068,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -100,
        "score": 18.910957021845874
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 26.281753907614387,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -40,
        "score": 49.31500357641914
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 26.281753907614387
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 13.128874563158115,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": -19.999999999999996,
        "score": 23.028344373033573
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 13.128874563158115
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": -1.2116300515052056,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 32.96085452633544
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": 2.8260596941021654,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -50,
        "score": 23.695570386654317
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 2.8260596941021654
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 62.8586809314782,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 53.092171826692656
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 62.8586809314782
      },
      "zh": {
       "accuracy": 0.2,
       "cc": -36.74882052545368,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": -33.333333333333336,
        "score": -75.21821298879041
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.632,
       "cc": 52.46751755678448,
       "cc_by_type": {
        "choice": 52.46751755678448
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 52.46751755678448
      },
      "noul": {
       "accuracy": 0.24,
       "cc": -52,
       "cc_by_type": {
        "noul": -52
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.45066666666666666,
       "cc": 17.650777067912205,
       "cc_by_type": {
        "score": 17.650777067912205
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 17.650777067912205
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -14.075490238875036,
       "cc_by_type": {
        "choice": 36,
        "noul": -100,
        "score": 21.77352928337489
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -16.706524469431017,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100,
        "score": 5.7810477097193775
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 20.95574183367787,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": -50,
        "score": 61.64771330591166
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 20.95574183367787
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -32.07708838513613,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": 9.031892739328462
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": -11.862692761495312,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100,
        "score": 22.38293620826768
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -11.447525637999656,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": -100,
        "score": 3.3932721426048085
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 4.739723573540634,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -71.42857142857143,
        "score": 34.667349992330585
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 4.739723573540634
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -19.95542954822079,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100,
        "score": 23.757404735128574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 5.930574972450304,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60,
        "score": 61.12505825068424
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 5.930574972450304
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": -0.20845375228950758,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": -66.66666666666667,
        "score": 30.281134103588304
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.36,
       "cc": 1.6257635079935728,
       "cc_by_type": {
        "choice": 65.11627906976744,
        "noul": -77.77777777777779,
        "score": 17.53878923199106
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 1.6257635079935728
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -5.984212815810622,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": 46.5634905848262
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 10.444007621596686,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -60,
        "score": 14.364652423331304
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 10.444007621596686
      },
      "model_routing": {
       "accuracy": 0.6082949308755761,
       "cc": 29.775167590225077,
       "cc_by_type": {
        "choice": 69.61038961038962,
        "noul": -6.060606060606055,
        "score": 25.77571922089167
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 29.775167590225077
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": -7.229669806205081,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 12.859934919196657
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 42.58687011240332,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 33.33333333333333,
        "score": 44.56655276989334
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 42.58687011240332
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": -2.6708439262919,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -100,
        "score": 21.54411846742971
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": -15.274838023083172,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -100,
        "score": 26.639254046692518
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.625,
       "cc": 20.18266992417227,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": -50,
        "score": 31.900322939776594
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 20.18266992417227
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.89481320849065,
      86.16359734422399
     ],
     "I": [
      21.17575994757959,
      28.329594719042426
     ],
     "capability": [
      51.150606303459746,
      56.002955246631714
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 23
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.35778065081391,
     "capability": 54.78583662123155,
     "intelligence": 25.21389259164919
    },
    "per_type": {
     "choice": {
      "accuracy": 0.632,
      "calibration": 84.17585650065509,
      "cc": 52.46751755678448,
      "cc_by_type": {
       "choice": 52.46751755678448
      },
      "cc_tiered": 57.805310036129775,
      "cc_tiered_open": 68.15046810038176,
      "cc_tiered_sealed": 47.46015197187779,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 52.46751755678448
     },
     "noul": {
      "accuracy": 0.24,
      "calibration": 81.95043662539807,
      "cc": -52,
      "cc_by_type": {
       "noul": -52
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.45066666666666666,
      "calibration": 86.9470488263886,
      "cc": 17.650777067912205,
      "cc_by_type": {
       "score": 17.650777067912205
      },
      "cc_tiered": 17.8363677388178,
      "cc_tiered_open": 19.23654465196982,
      "cc_tiered_sealed": 16.436190825665776,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 17.650777067912205
     }
    },
    "raw_on_A": {
     "calibration": 77.52017651507438,
     "capability": 50.95601359015508,
     "intelligence": 24.391850665235772
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Celeris",
   "axes": {
    "calibration": 88.35471243386242,
    "cost": 60.373088134904265,
    "intelligence": 58.890192862872325,
    "speed": 86.92599216009037
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05393650793650812,
      "ece_score": 89.21269841269837,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 89.21269841269837,
      "score_before_time_limit": 89.21269841269837,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.21333333333333337,
      "brier": 0.10485600000000002,
      "ece": 0.06373333333333334,
      "n": 375,
      "score": 87.25333333333334,
      "score_before_time_limit": 87.25333333333334,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9093333333333333,
      "interval90_width": 0.25799999999999995,
      "n": 375,
      "nrps": 0.07539788888888894,
      "score": 88.59810555555555,
      "score_before_time_limit": 88.59810555555555,
      "time_limit_failed_share": 0,
      "top_ece": 0.07632000000000008
     }
    },
    "score": 88.35471243386242
   },
   "capability": 73.62245264836737,
   "composite_ci95": {
    "A": [
     68.60424952368858,
     72.50734096171499
    ],
    "B": [
     64.96346923338149,
     70.52759161704832
    ],
    "C": [
     54.81823135353137,
     72.50734096171499
    ]
   },
   "cost": {
    "basis": "Celeris public price, USD 0.04/M input, output free (celeris.ai/pricing, fetched 16 Oct 2026 19:27Z, saved copy sha256 c3ed0fdc...; docs.celeris.ai/pricing rates table, saved copy sha256 ff4b658d..., byte-identical to the copy saved on 14 Oct 2026): \"Celeris-1 Decision $0.04 / M input tokens, Free output tokens\"; prepaid per-token billing with no subscription or commitment. Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap, d1-ultra, Sage 1.4, Nimble and kettei), through a scoped re-freeze of the frozen pricing module that adds this one key and nothing else, with a zero-difference proof for every other registry entry. Workspace activation at Celeris is capacity-limited (an activation waitlist, as for SubQ Snap); free or promotional credit is not used. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (1,968,382 input tokens over the 1,500 answered items, 0 output tokens), divided by the 1,485 items of the common cost basis. Celeris counts input tokens its own way: per its docs, input_tokens \"covers everything the model reads for the request, which is more than your state and questions alone\"; on the same 1,500 items it reports about 3.0x the input tokens kettei-fat reports, so the cost reflects Celeris's own count. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.020936 per 1,000 decisions (0.324x the official cost cap USD 0.064595), against USD 0.052490 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05249018666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020936161616161617
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Celeris-1 Decision (Celeris)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted inference.celeris.ai/celeris-1-decision System One API (POST <base>/v1/systemone), model `celeris-1-decision`, through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `celeris` (no proxy) before the first request (under the pre-registration). Native probabilities on all three question types; Celeris returns every probability at two-decimal precision (1,500 of 1,500 rows; 332 returned probabilities are exactly 0), scored as returned. The runner recorded celeris-1-decision as the answering model on all 1,500 rows (the adapter reads the response's model field). All 1,500 items answered with HTTP 200, including the 15 long items, which fit the documented 131,072-token context window (largest reported input 81,719 tokens). paid evaluation, run with the customer's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 62.630814168468525,
    "I_sealed": 55.14957155727612,
    "base": 58.890192862872325,
    "excess": -2.358284159163304,
    "gap": 7.481242611192407,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.52874311900909,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 71.6892104435357,
       "judge": 76.26895187870798,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 54.45410628019323,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 19.999999999999996,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 57.909593106203275,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 70.42424242424244,
       "hard": 61.16666666666667,
       "judge": 58.91525423728814,
       "standard": 43.629629629629605
      }
     },
     "sealed|choice": {
      "cc": 74.18524524536124,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 68.77529779114143,
       "judge": 72.67384916748287,
       "standard": 76.2708289390424
      }
     },
     "sealed|noul": {
      "cc": 40.147216653650005,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 58.62068965517242,
       "hard": 25.454545454545464,
       "judge": 41.53846153846155,
       "standard": 58.20895522388059
      }
     },
     "sealed|score": {
      "cc": 51.11625277281713,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 67.62280701754383,
       "hard": 50.50985915492958,
       "judge": 46.203125,
       "standard": 51.445454545454524
      }
     }
    }
   },
   "jevbench_score": 70.96415918238667,
   "key": "celeris-1-decision",
   "last_measured_on": "2026-10-16",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.25",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:inference.celeris.ai/celeris-1-decision/v1/systemone#celeris-1-decision",
   "noul_decisive": {
    "acc_among_decisive": 0.9254237288135593,
    "decisive": 295,
    "decisive_rate": 0.7866666666666666,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 13,
    "axes": {
     "calibration": 88.35471243386242,
     "cost": 60.373088134904265,
     "intelligence": 58.890192862872325,
     "speed": 86.97194897947064
    },
    "board_release": "v1.6.43",
    "capability": 73.62245264836737,
    "capability_api_board_rank": 14,
    "capability_open_board_rank": null,
    "capability_rank": 28,
    "composite_ci95": {
     "A": [
      68.61140285654639,
      72.51533144283196
     ],
     "B": [
      64.96860049103343,
      70.53363954277505
     ],
     "C": [
      54.823943816415806,
      72.51533144283196
     ]
    },
    "jevbench_score": 70.97181314040431,
    "measured_in": "v1.6.25",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 20,
    "ranks": {
     "A": 20,
     "B": 28,
     "C": 30,
     "capability": 52
    },
    "scores": {
     "A": 70.97181314040431,
     "B": 68.174542834176,
     "C": 68.3705939780874
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.4326209495193325,
     "p50_s_raw": 0.4326209495193325,
     "p95_s_adjusted": 0.46419189785665366,
     "p95_s_raw": 0.46419189785665366
    },
    "views": {
     "Balanced 33:33:33": 66.60392918110978,
     "Emphasis on Accuracy 60:20:20": 63.28801502769513,
     "Emphasis on Cost 20:20:60": 63.96337756642816,
     "Emphasis on Speed 20:60:20": 73.48801217314741,
     "Intelligence only": 58.89019286287232
    }
   },
   "rank": 68,
   "ranked": true,
   "ranks": {
    "A": 68,
    "B": 68,
    "C": 66,
    "capability": 70
   },
   "repo": "https://docs.celeris.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.96415918238667,
    "B": 68.16889268941678,
    "C": 68.36322054865403
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.43277494149515405,
    "p50_s_raw": 0.43277494149515405,
    "p95_s_adjusted": 0.46896310617739795,
    "p95_s_raw": 0.46896310617739795
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 88.55555555555556,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 74
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 88.55555555555556
      },
      "calibration_gen": {
       "accuracy": 0.7029702970297029,
       "cc": 56.85813229056203,
       "cc_by_type": {
        "choice": 77.7306468716861,
        "noul": 36.00000000000001,
        "score": 56.84374999999999
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 56.85813229056203
      },
      "composite_gen": {
       "accuracy": 0.8613861386138614,
       "cc": 69.98257012484656,
       "cc_by_type": {
        "choice": 86.9918699186992,
        "noul": 48.14814814814814,
        "score": 74.80769230769232
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 69.98257012484656
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.5925925925926,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 96.77777777777777
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.5925925925926
      },
      "jaggedness": {
       "accuracy": 0.6222222222222222,
       "cc": 36.09978545606696,
       "cc_by_type": {
        "choice": 59.57651588065449,
        "noul": 18.918918918918926,
        "score": 29.803921568627445
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 36.09978545606696
      },
      "llm_router": {
       "accuracy": 0.8083067092651757,
       "cc": 68.70445180122648,
       "cc_by_type": {
        "choice": 74.64119045522585,
        "noul": 69.0721649484536,
        "score": 62.40000000000001
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.70445180122648
      },
      "long_state": {
       "accuracy": 0.4666666666666667,
       "cc": -28.00537634408603,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -19.50000000000003
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5185185185185185,
       "cc": 14.397316314842081,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -25,
        "score": 30.047619047619033
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 14.397316314842081
      },
      "multilingual": {
       "accuracy": 0.7163636363636363,
       "cc": 56.70515248138471,
       "cc_by_type": {
        "choice": 70.6553635474405,
        "noul": 41.66666666666667,
        "score": 57.793427230046944
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 56.70515248138471
      },
      "negation_pair_gen": {
       "accuracy": 0.7674418604651163,
       "cc": 50.21111111111111,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 19.999999999999996,
        "score": 37.3
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 50.21111111111111
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 82.92889042889043,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100,
        "score": 69.8076923076923
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 82.92889042889043
      },
      "usecase": {
       "accuracy": 0.7045454545454546,
       "cc": 52.40729411764707,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 41.66666666666667,
        "score": 42.51600000000001
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 52.40729411764707
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 10.50061050061049,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50,
        "score": -42.307692307692335
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 10.50061050061049
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 46.74444444444443,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 6.899999999999984
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 46.74444444444443
      },
      "de": {
       "accuracy": 0.875,
       "cc": 87.51365909965371,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 86.43478260869566
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.51365909965371
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 54.66014897579143,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 97.49999999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 54.66014897579143
      },
      "en": {
       "accuracy": 0.7654213727193745,
       "cc": 59.11116852992231,
       "cc_by_type": {
        "choice": 76.71495344211078,
        "noul": 46.76258992805755,
        "score": 53.85596221959859
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 59.11116852992231
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 59.20020471483863,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 75,
        "score": 22.176470588235276
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 59.20020471483863
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 22.102003642987246,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 97.99999999999999
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 22.102003642987246
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 84.71310629514964,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 75.31578947368422
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.71310629514964
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 51.663407821229065,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 25,
        "score": 46.750000000000014
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 51.663407821229065
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 71.551683183016,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 42.71428571428573
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.551683183016
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 43.147383284793925,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 39.99999999999999,
        "score": 32.33333333333333
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 43.147383284793925
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 23.642857142857128,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 70.9285714285714
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 23.642857142857128
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.16666666666667,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 98.16666666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.16666666666667
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 33.964213564213566,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 61.19999999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 33.964213564213566
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 56.32830097071453,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 39.99999999999999,
        "score": 51.83720930232556
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 56.32830097071453
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 50.6765829642443,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 60.00000000000001,
        "score": 45.06451612903225
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 50.6765829642443
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 63.15602836879433,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 55.00000000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 63.15602836879433
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 65.375,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 46.12500000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 65.375
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.99462365591397,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 92.49999999999999
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.99462365591397
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 46.111111111111114,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 5.000000000000004
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.111111111111114
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8093333333333333,
       "cc": 75.37266308195717,
       "cc_by_type": {
        "choice": 75.37266308195717
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 75.37266308195717
      },
      "noul": {
       "accuracy": 0.728,
       "cc": 45.599999999999994,
       "cc_by_type": {
        "noul": 45.599999999999994
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 45.599999999999994
      },
      "score": {
       "accuracy": 0.6693333333333333,
       "cc": 53.62184115523472,
       "cc_by_type": {
        "score": 53.62184115523472
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 53.62184115523472
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": -3.3333333333333335,
       "cc_by_type": {
        "choice": 84,
        "noul": -100,
        "score": 5.999999999999995
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 60.539178213091255,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 100,
        "score": 18.884615384615376
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 60.539178213091255
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 53.159891598916,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 41.99999999999999
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 53.159891598916
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -5.883528265107214,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": 9.250000000000004
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 38.08121980676328,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 76.56249999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 38.08121980676328
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 25.78092243186579,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 100,
        "score": -97.5000000000001
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 25.78092243186579
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 34.35667600373482,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 14.28571428571428,
        "score": 27.999999999999993
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 34.35667600373482
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 25.161793667626117,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": -50,
        "score": 46.39130434782608
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 25.161793667626117
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": -0.9074074074074095,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 19.77777777777777
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 35.525944671554946,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 24.636363636363644
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 35.525944671554946
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 53.53648892020985,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 16.681818181818187
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 53.53648892020985
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 61.327956989247305,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 48.499999999999986
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 61.327956989247305
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 64.07245681381958,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 60.00000000000001,
        "score": 55.24999999999998
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 64.07245681381958
      },
      "model_routing": {
       "accuracy": 0.8755760368663594,
       "cc": 75.8801857864358,
       "cc_by_type": {
        "choice": 89.48051948051949,
        "noul": 87.87878787878789,
        "score": 50.28125000000002
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.8801857864358
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 19.180836250047037,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 69.05882352941175
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 19.180836250047037
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 58.28367021403233,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 58.23076923076924
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 58.28367021403233
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 27.49618468076886,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0,
        "score": 26.823529411764714
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 27.49618468076886
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 54.415942028985505,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 58.9
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 54.415942028985505
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.34210526315788,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 83.02631578947367
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.34210526315788
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.97879030820103,
      89.7941902777778
     ],
     "I": [
      53.60515070087634,
      63.490315354109036
     ],
     "capability": [
      69.55833052839986,
      75.84428813904887
     ],
     "composite": {
      "A": [
       68.60424952368858,
       72.50734096171499
      ],
      "B": [
       64.96346923338149,
       70.52759161704832
      ],
      "C": [
       54.81823135353137,
       72.50734096171499
      ]
     },
     "seed": 24
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.35471243386242,
     "capability": 73.62245264836737,
     "intelligence": 58.890192862872325
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8093333333333333,
      "calibration": 89.21269841269837,
      "cc": 75.37266308195717,
      "cc_by_type": {
       "choice": 75.37266308195717
      },
      "cc_tiered": 74.85699418218516,
      "cc_tiered_open": 75.52874311900909,
      "cc_tiered_sealed": 74.18524524536124,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.37266308195717
     },
     "noul": {
      "accuracy": 0.728,
      "calibration": 87.25333333333334,
      "cc": 45.599999999999994,
      "cc_by_type": {
       "noul": 45.599999999999994
      },
      "cc_tiered": 47.30066146692162,
      "cc_tiered_open": 54.45410628019323,
      "cc_tiered_sealed": 40.147216653650005,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 45.599999999999994
     },
     "score": {
      "accuracy": 0.6693333333333333,
      "calibration": 88.59810555555555,
      "cc": 53.62184115523472,
      "cc_by_type": {
       "score": 53.62184115523472
      },
      "cc_tiered": 54.512922939510204,
      "cc_tiered_open": 57.909593106203275,
      "cc_tiered_sealed": 51.11625277281713,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.62184115523472
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.59494163888954,
    "Emphasis on Accuracy 60:20:20": 63.28314579381391,
    "Emphasis on Cost 20:20:60": 63.95840386047157,
    "Emphasis on Speed 20:60:20": 73.46832021219848,
    "Intelligence only": 58.89019286287232
   }
  },
  {
   "adapter_id": "images/certo/spec.json:certo_v15.CertoLocalV15Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "AltSlate Labs",
   "axes": {
    "calibration": 86.14595667337036,
    "cost": 96.9130801359906,
    "intelligence": 0.20278658812856143,
    "speed": 92.44797254495796
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06436937500000006,
      "ece_score": 87.12612499999999,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 87.12612499999999,
      "score_before_time_limit": 87.12612499999999,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.9893333333333333,
      "brier": 0.25290737730666696,
      "ece": 0.08273039999999988,
      "n": 375,
      "score": 83.45392000000002,
      "score_before_time_limit": 83.45392000000002,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9946666666666667,
      "interval90_width": 0.9942222222222223,
      "n": 375,
      "nrps": 0.17416936626444454,
      "score": 87.8578250201111,
      "score_before_time_limit": 87.8578250201111,
      "time_limit_failed_share": 0,
      "top_ece": 0.0343370666666668
     }
    },
    "score": 86.14595667337036
   },
   "capability": 43.17437163074946,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0,
     0.00442750939229265
    ],
    "B": [
     0,
     0.0028324122193567084
    ],
    "C": [
     0,
     0.0030746593002032303
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0012673522167487683
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Certo v1 (AltSlate Labs)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 0.12963682017919345,
    "I_sealed": 0.27593635607792943,
    "base": 0.20278658812856143,
    "excess": -2.71968717833698,
    "gap": -0.14629953589873598,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 0,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486491,
       "hard": 1.855929537590448,
       "judge": -18.655240606460122,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100,
       "hard": -91.30434782608697,
       "judge": -100,
       "standard": -100
      }
     },
     "open|score": {
      "cc": 0.3889104605375804,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 5.111515151515156,
       "hard": -0.6524999999999892,
       "judge": 1.8106779661016925,
       "standard": -2.0222222222222364
      }
     },
     "sealed|choice": {
      "cc": 0.6168084160817238,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -6.42228955731281,
       "hard": -4.86425339366518,
       "judge": -0.5586592178770757,
       "standard": 16.861682473210998
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "sealed|score": {
      "cc": 0.21100065215206465,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -0.7955319148936102,
       "hard": 1.9993137254901705,
       "judge": -1.0858682634730998,
       "standard": -0.9170558375635629
      }
     }
    }
   },
   "jevbench_score": 1.3254504525307007e-05,
   "key": "certo",
   "last_measured_on": "2026-10-02",
   "licence": "MIT",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.25,
    "decisive_rate": 0.010666666666666666,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.14595667337036,
     "cost": 96.9130801359906,
     "intelligence": 0.20278658812856143,
     "speed": 92.53493469610669
    },
    "board_release": "v1.6.43",
    "capability": 43.17437163074946,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 106,
    "capability_rank": 134,
    "composite_ci95": {
     "A": [
      0,
      0.004427570625789243
     ],
     "B": [
      0,
      0.0028324322673094096
     ],
     "C": [
      0,
      0.003074701823464752
     ]
    },
    "jevbench_score": 1.3254531668229452e-05,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 192,
    "rank": 236,
    "ranks": {
     "A": 236,
     "B": 236,
     "C": 236,
     "capability": 195
    },
    "scores": {
     "A": 1.3254531668229452e-05,
     "B": 8.311485824043969e-06,
     "C": 9.204535880714898e-06
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23407380385324358,
     "p50_s_raw": 0.042036901926621795,
     "p95_s_adjusted": 0.2383162835612893,
     "p95_s_raw": 0.04415814178064466
    },
    "views": {
     "Balanced 33:33:33": 9.964199692814982e-06,
     "Emphasis on Accuracy 60:20:20": 5.551453514139297e-06,
     "Emphasis on Cost 20:20:60": 1.653808420915537e-05,
     "Emphasis on Speed 20:60:20": 1.6534837758955257e-05,
     "Intelligence only": 3.3356285030110653e-06
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 277,
   "ranked": true,
   "ranks": {
    "A": 277,
    "B": 277,
    "C": 277,
    "capability": 235
   },
   "repo": "https://huggingface.co/altslate/certo-decision-model",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 1.3254504525307007e-05,
    "B": 8.311477285661616e-06,
    "C": 9.204517031463201e-06
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23454320020973682,
    "p50_s_raw": 0.04227160010486841,
    "p95_s_adjusted": 0.24264977835118767,
    "p95_s_raw": 0.046324889175593846
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "ModernBERT-large with a per-option query/scoring head, ~400M parameters",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.84167298672317,
      89.02661690205646
     ],
     "I": [
      0,
      1.4261709005081855
     ],
     "capability": [
      40.54746106861931,
      44.91215541214051
     ],
     "composite": {
      "A": [
       0,
       0.00442750939229265
      ],
      "B": [
       0,
       0.0028324122193567084
      ],
      "C": [
       0,
       0.0030746593002032303
      ]
     },
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.14595667337036,
     "capability": 43.17437163074946,
     "intelligence": 0.20278658812856143
    },
    "per_type": {
     "choice": {
      "accuracy": 0.21733333333333332,
      "calibration": 87.12612499999999,
      "cc": -0.09188825630196516,
      "cc_by_type": {
       "choice": -0.09188825630196516
      },
      "cc_tiered": 0.3084042080408619,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0.6168084160817238,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0
     },
     "noul": {
      "accuracy": 0.0026666666666666666,
      "calibration": 83.45392000000002,
      "cc": -99.46666666666667,
      "cc_by_type": {
       "noul": -99.46666666666667
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.27466666666666667,
      "calibration": 87.8578250201111,
      "cc": 0.4315701754385892,
      "cc_by_type": {
       "score": 0.4315701754385892
      },
      "cc_tiered": 0.2999555563448225,
      "cc_tiered_open": 0.3889104605375804,
      "cc_tiered_sealed": 0.21100065215206465,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.4315701754385892
     }
    },
    "raw_on_A": {
     "calibration": 84.87407902121794,
     "capability": 42.46944871565377,
     "intelligence": 0.06481841008959673
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 9.964179240079037e-06,
    "Emphasis on Accuracy 60:20:20": 5.551449704956405e-06,
    "Emphasis on Cost 20:20:60": 1.6538050403588868e-05,
    "Emphasis on Speed 20:60:20": 1.6534736382482662e-05,
    "Intelligence only": 3.3356285030110653e-06
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "chaoliangUNSW",
   "axes": {
    "calibration": 88.50529260806339,
    "cost": 90.98733169608725,
    "intelligence": 33.91785114475573,
    "speed": 94.44817145387303
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.045939073484389595,
      "ece_score": 90.81218530312208,
      "mean_tvd": null,
      "n": 746,
      "n_ece_hard": 350,
      "n_tvd": 0,
      "score": 90.81218530312208,
      "score_before_time_limit": 90.81218530312208,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.5106951871657754,
      "brier": 0.16370798403819561,
      "ece": 0.0612531783435275,
      "n": 374,
      "score": 87.7493643312945,
      "score_before_time_limit": 87.7493643312945,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9173333333333333,
      "interval90_width": 0.36755555555555564,
      "n": 375,
      "nrps": 0.09314457158564356,
      "score": 86.95432818977358,
      "score_before_time_limit": 86.95432818977358,
      "time_limit_failed_share": 0,
      "top_ece": 0.08388443230944237
     }
    },
    "score": 88.50529260806339
   },
   "capability": 61.21157187640956,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 491,898 input + 0 output tokens over 1,489 decisions = 330.354600403 tokens/decision. GPU-time cost USD 0.0019972033083885/1,000; old list-price cost USD 0.0066070920080591/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0019972033083885/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.006589159999999999
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0019972033083885
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of Jev-Style-2B, a full fine-tune of Qwen/Qwen3.5-2B (Apache-2.0). TRAINING: trained in part on soft labels distilled from TypeSafe Jev 1.14 via its public API (334,739 rows), plus official train splits of several public benchmarks (not zero-shot); the author states that public JevBench items were not used. We did not inspect the training examples or verify overlap. SERVING: measured through the author's own documented HTTP server on CUDA, bf16, CUDA graphs and the FLA kernel; the author's own 74.9% JevBench figure used float32 on Apple MPS and is not an official result. The multimodal tower is present but unused on this text draw. Contributor code ran only on a disposable rented pod. ITEMS: all 1,500 items of the draw were sent; the 5 longest items exceed the model's 32,768-token input limit and were refused (HTTP 400), scored as misses and kept in the denominator. TOKENS: output tokens reported as 0 (native probability distribution readout, no text generation); input is the server's native usage. DRAW: measured on the 27 Oct 2026 regular draw (fresh sealed items; not yet equated to the reference scale, so its numbers are shown as published, offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own draw. LATENCY: p50 0.013 s, p95 0.027 s, maximum 0.089 s.",
   "display": "Jev-Style-2B",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-28",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.17",
    "results_sha256": "e970d8fb5adcdbdf164859e8faac65ff1a7825e83210b80bc2375ec9c2414ba0",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Self-hosted on one evaluator-rented Verda RTX 6000 Ada GPU in Finland (EU), through the author's own documented HTTP server: CUDA, bf16, CUDA graphs and FLA kernel. Measured 28 Oct 2026, 16:45-16:56 UTC. Contributor code ran only on a disposable rented pod, with network cut before the scored run. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 38.304261084071186,
    "I_sealed": 29.53144120544027,
    "base": 33.91785114475573,
    "excess": 6.199432236192672,
    "gap": 8.772819878630916,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 67.99374492407335,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 54.70273670965714,
       "judge": 84.17930125247199,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": 4.347826086956519,
       "judge": -30.000000000000004,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 46.91903832814022,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.1198524735361,
       "hard": 40.77752386140586,
       "judge": 54.6366915185271,
       "standard": 43.025180403330666
      }
     },
     "sealed|choice": {
      "cc": 52.61089225060495,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 80.34719947592534,
       "hard": 42.00295795478555,
       "judge": 54.4434857635893,
       "standard": 57.20971696010699
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 21.839080459770123,
       "hard": -58.53658536585367,
       "judge": -11.111111111111116,
       "standard": -11.627906976744185
      }
     },
     "sealed|score": {
      "cc": 35.983431365715866,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 63.71695161559996,
       "hard": 17.043082174173918,
       "judge": 45.394746410356554,
       "standard": 45.88039705689669
      }
     }
    }
   },
   "jevbench_score": 29.516830172016306,
   "key": "chaoliangunsw-jev-style-2b-decision-v4",
   "last_measured_on": "2026-10-28",
   "licence": "Apache-2.0",
   "listing": "unranked",
   "measured_in": "v1.7.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": {
    "base": "Qwen/Qwen3.5-2B",
    "model": "chaoliangUNSW/Jev-Style-2B-Decision-v4",
    "revision": "c9504148dae7f7a43d67a5782f12a015a4b4022c",
    "served_model": "jev-style-2b-decision-v4"
   },
   "new_in": "v1.7.17",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale; stronger anchor runs for that draw are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.907103825136612,
    "decisive": 183,
    "decisive_rate": 0.4893048128342246,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "provider": "self-hosted",
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/chaoliangUNSW/Jev-Style-2B-Decision-v4",
   "scores": {
    "A": 29.516830172016306,
    "B": 25.051863606761444,
    "C": 20.497798730566885
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.17682074799997735,
    "p50_s_raw": 0.013410373999988678,
    "p95_s_adjusted": 0.20307180599997993,
    "p95_s_raw": 0.02653590299998996
   },
   "status": {
    "answered_ok": 1495,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.75,
       "cc": 66.66666666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 66.66666666666666
      },
      "calibration_gen": {
       "accuracy": 0.42857142857142855,
       "cc": -13.71050841063181,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 10.816526716156517
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.47368421052631576,
       "cc": 20.724219635062312,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -19.999999999999996,
        "score": 8.839325571853596
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 20.724219635062312
      },
      "fairness": {
       "accuracy": 0.8571428571428571,
       "cc": 65.4720083302664,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 96.41602499079919
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 65.4720083302664
      },
      "jaggedness": {
       "accuracy": 0.5454545454545454,
       "cc": -3.5998322839015295,
       "cc_by_type": {
        "choice": 38.2716049382716,
        "noul": -100,
        "score": 50.92889821002382
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.6888888888888889,
       "cc": 49.48357499253626,
       "cc_by_type": {
        "choice": 65.6286721504113,
        "noul": 30.434782608695656,
        "score": 52.38727021850181
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 49.48357499253626
      },
      "long_state": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0,
       "cc": -66.66666666666667,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5808903365906624,
       "cc": 26.673771393572796,
       "cc_by_type": {
        "choice": 54.08493209743482,
        "noul": -15.573770491803273,
        "score": 41.510152575086835
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 26.673771393572796
      },
      "negation_pair_gen": {
       "accuracy": 0.6363636363636364,
       "cc": 33.33333333333333,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 33.33333333333333
      },
      "option_order": {
       "accuracy": 0.7777777777777778,
       "cc": 71.42857142857144,
       "cc_by_type": {
        "choice": 71.42857142857144
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 71.42857142857144
      },
      "perturb_pair": {
       "accuracy": 0.7857142857142857,
       "cc": 61.11067555696246,
       "cc_by_type": {
        "choice": 90.56603773584905,
        "noul": 19.999999999999996,
        "score": 72.76598893503834
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 61.11067555696246
      },
      "usecase": {
       "accuracy": 0.5784615384615385,
       "cc": 29.368275257832977,
       "cc_by_type": {
        "choice": 46.754194018964235,
        "noul": -4.61538461538461,
        "score": 45.9660163699193
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 29.368275257832977
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6666666666666666,
       "cc": 44.19662838277184,
       "cc_by_type": {
        "choice": 58.904109589041084,
        "noul": 0,
        "score": 73.68577555927443
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 44.19662838277184
      },
      "cs": {
       "accuracy": 0.6216216216216216,
       "cc": 20.625428116690852,
       "cc_by_type": {
        "choice": 67.91443850267379,
        "noul": -55.55555555555556,
        "score": 49.51740140295432
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 20.625428116690852
      },
      "da": {
       "accuracy": 0.5833333333333334,
       "cc": 21.313812875901167,
       "cc_by_type": {
        "choice": 55.27950310559007,
        "noul": -27.27272727272727,
        "score": 35.93466279484071
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 21.313812875901167
      },
      "de": {
       "accuracy": 0.625,
       "cc": 38.34676533688945,
       "cc_by_type": {
        "choice": 53.69179315667609,
        "noul": 25,
        "score": 36.34850285399225
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 38.34676533688945
      },
      "el": {
       "accuracy": 0.42857142857142855,
       "cc": 1.9445334294158438,
       "cc_by_type": {
        "choice": 42.307692307692314,
        "noul": -60,
        "score": 23.525907980555218
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 1.9445334294158438
      },
      "en": {
       "accuracy": 0.6144144144144145,
       "cc": 32.73650784520552,
       "cc_by_type": {
        "choice": 56.16883116883115,
        "noul": -6.5573770491803245,
        "score": 48.59806941596575
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 32.73650784520552
      },
      "es": {
       "accuracy": 0.6216216216216216,
       "cc": 35.971346199356255,
       "cc_by_type": {
        "choice": 85.56535685645551,
        "noul": 33.33333333333333,
        "score": -10.984651591720063
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 35.971346199356255
      },
      "fi": {
       "accuracy": 0.40384615384615385,
       "cc": 10.907513560718826,
       "cc_by_type": {
        "choice": 24.571428571428577,
        "noul": -36.8421052631579,
        "score": 44.9932173738858
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 10.907513560718826
      },
      "fr": {
       "accuracy": 0.53125,
       "cc": 27.7867034371714,
       "cc_by_type": {
        "choice": 59.18367346938776,
        "noul": -11.111111111111116,
        "score": 35.28754795323755
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 27.7867034371714
      },
      "hi": {
       "accuracy": 0.35135135135135137,
       "cc": -6.9229707839524695,
       "cc_by_type": {
        "choice": 25.619834710743806,
        "noul": -77.77777777777779,
        "score": 31.38903071517657
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.6585365853658537,
       "cc": 41.90415051463105,
       "cc_by_type": {
        "choice": 53.91705069124423,
        "noul": 16.666666666666675,
        "score": 55.12873418598223
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 41.90415051463105
      },
      "it": {
       "accuracy": 0.6888888888888889,
       "cc": 42.2290212797247,
       "cc_by_type": {
        "choice": 52.0851818988465,
        "noul": 16.666666666666675,
        "score": 57.93521527366092
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 42.2290212797247
      },
      "ja": {
       "accuracy": 0.5454545454545454,
       "cc": 14.143548484951713,
       "cc_by_type": {
        "choice": 61.139896373057,
        "noul": -27.27272727272727,
        "score": 8.563476354525413
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 14.143548484951713
      },
      "ko": {
       "accuracy": 0.5348837209302325,
       "cc": 9.13847658261102,
       "cc_by_type": {
        "choice": 55.88235294117647,
        "noul": -60,
        "score": 31.53307680665658
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 9.13847658261102
      },
      "mixed": {
       "accuracy": 0.734375,
       "cc": 57.020378328721584,
       "cc_by_type": {
        "choice": 72.997299729973,
        "noul": 50,
        "score": 48.06383525619176
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 57.020378328721584
      },
      "nl": {
       "accuracy": 0.5555555555555556,
       "cc": 9.698450877935104,
       "cc_by_type": {
        "choice": 50.47169811320755,
        "noul": -60,
        "score": 38.62365452059776
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 9.698450877935104
      },
      "no": {
       "accuracy": 0.6078431372549019,
       "cc": 27.414828438517088,
       "cc_by_type": {
        "choice": 53.52839931153184,
        "noul": -6.666666666666665,
        "score": 35.38275267068609
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 27.414828438517088
      },
      "pl": {
       "accuracy": 0.5789473684210527,
       "cc": 28.80432629351807,
       "cc_by_type": {
        "choice": 47.31240021287918,
        "noul": -5.882352941176472,
        "score": 44.982931608851494
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 28.80432629351807
      },
      "pt": {
       "accuracy": 0.5531914893617021,
       "cc": 25.778289362446476,
       "cc_by_type": {
        "choice": 40.099833610648915,
        "noul": 0,
        "score": 37.23503447669051
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 25.778289362446476
      },
      "sv": {
       "accuracy": 0.6521739130434783,
       "cc": 28.68989808998238,
       "cc_by_type": {
        "choice": 73.18435754189943,
        "noul": -50,
        "score": 62.885336728047704
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 28.68989808998238
      },
      "tr": {
       "accuracy": 0.5,
       "cc": 15.67579051922444,
       "cc_by_type": {
        "choice": 58.950969213226905,
        "noul": -60,
        "score": 48.07640234444642
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 15.67579051922444
      },
      "uk": {
       "accuracy": 0.5535714285714286,
       "cc": 27.47130917893534,
       "cc_by_type": {
        "choice": 34.24657534246575,
        "noul": 0,
        "score": 48.16735219434026
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 27.47130917893534
      },
      "zh": {
       "accuracy": 0.7073170731707317,
       "cc": 57.45079991694757,
       "cc_by_type": {
        "choice": 62.80991735537191,
        "noul": 50,
        "score": 59.54248239547078
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 57.45079991694757
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6693333333333333,
       "cc": 54.89587859069825,
       "cc_by_type": {
        "choice": 54.89587859069825
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 54.89587859069825
      },
      "noul": {
       "accuracy": 0.44266666666666665,
       "cc": -11.466666666666669,
       "cc_by_type": {
        "noul": -11.466666666666669
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.592,
       "cc": 43.697546320368,
       "cc_by_type": {
        "score": 43.697546320368
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 43.697546320368
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4583333333333333,
       "cc": 2.552378839266519,
       "cc_by_type": {
        "choice": 33.283914010378055,
        "noul": -71.42857142857143,
        "score": 45.80179393599293
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 2.552378839266519
      },
      "code_linting": {
       "accuracy": 0.463768115942029,
       "cc": 4.194542918912137,
       "cc_by_type": {
        "choice": 37.53827311696264,
        "noul": -33.333333333333336,
        "score": 8.378688973107106
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 4.194542918912137
      },
      "customer_support": {
       "accuracy": 0.6666666666666666,
       "cc": 28.80777198688607,
       "cc_by_type": {
        "choice": 74.16974169741697,
        "noul": -46.666666666666664,
        "score": 58.920240929907905
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 28.80777198688607
      },
      "demand_forecasting": {
       "accuracy": 0.45454545454545453,
       "cc": -6.33870571666112,
       "cc_by_type": {
        "choice": 49.96209249431388,
        "noul": -100,
        "score": 31.021790355702763
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5645161290322581,
       "cc": 19.574614400464032,
       "cc_by_type": {
        "choice": 57.77621393384941,
        "noul": -41.17647058823529,
        "score": 42.12409985577798
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 19.574614400464032
      },
      "feature_extraction": {
       "accuracy": 0.5972222222222222,
       "cc": 28.59113531861235,
       "cc_by_type": {
        "choice": 51.005025125628144,
        "noul": 14.28571428571428,
        "score": 20.48266654449463
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 28.59113531861235
      },
      "financial_crime": {
       "accuracy": 0.5555555555555556,
       "cc": 10.3500656302977,
       "cc_by_type": {
        "choice": 48.963317384370015,
        "noul": -63.63636363636363,
        "score": 45.723243142886716
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 10.3500656302977
      },
      "gaming": {
       "accuracy": 0.603448275862069,
       "cc": 32.84108126953017,
       "cc_by_type": {
        "choice": 56.97211155378488,
        "noul": -11.111111111111116,
        "score": 52.66224336591675
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 32.84108126953017
      },
      "insurance_claims": {
       "accuracy": 0.5483870967741935,
       "cc": 16.951580792195895,
       "cc_by_type": {
        "choice": 42.09354120267261,
        "noul": -17.647058823529417,
        "score": 26.408259997444482
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 16.951580792195895
      },
      "knowledge_graphs": {
       "accuracy": 0.5074626865671642,
       "cc": -2.2353537814133397,
       "cc_by_type": {
        "choice": 46.026986506746624,
        "noul": -69.23076923076923,
        "score": 16.497721379782583
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.6533333333333333,
       "cc": 40.82619361899566,
       "cc_by_type": {
        "choice": 57.686882933709455,
        "noul": 19.999999999999996,
        "score": 44.79169792327752
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 40.82619361899566
      },
      "legal_compliance": {
       "accuracy": 0.6428571428571429,
       "cc": 34.86254582856723,
       "cc_by_type": {
        "choice": 63.89167502507522,
        "noul": 14.28571428571428,
        "score": 26.410248174912187
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 34.86254582856723
      },
      "llm_guardrails": {
       "accuracy": 0.6338028169014085,
       "cc": 37.937307796827504,
       "cc_by_type": {
        "choice": 63.34012219959267,
        "noul": 23.076923076923084,
        "score": 27.39487811396677
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 37.937307796827504
      },
      "model_routing": {
       "accuracy": 0.6554054054054054,
       "cc": 39.37877429795244,
       "cc_by_type": {
        "choice": 62.61035139345683,
        "noul": 5.882352941176472,
        "score": 49.643618559224024
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 39.37877429795244
      },
      "moderation_trust_safety": {
       "accuracy": 0.5084745762711864,
       "cc": 18.818852416390182,
       "cc_by_type": {
        "choice": 34.640522875817,
        "noul": -17.647058823529417,
        "score": 39.46309319688296
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 18.818852416390182
      },
      "other": {
       "accuracy": 0.46875,
       "cc": 9.085490545948543,
       "cc_by_type": {
        "choice": 26.748057713651498,
        "noul": 0,
        "score": 0.5084139241941288
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 9.085490545948543
      },
      "recruiting": {
       "accuracy": 0.5692307692307692,
       "cc": 30.920775814118088,
       "cc_by_type": {
        "choice": 48.089401586157166,
        "noul": 4.0000000000000036,
        "score": 40.672925856197104
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 30.920775814118088
      },
      "risk_assessment": {
       "accuracy": 0.5416666666666666,
       "cc": 18.187291319158085,
       "cc_by_type": {
        "choice": 37.23849372384937,
        "noul": -23.076923076923073,
        "score": 40.40030331054795
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 18.187291319158085
      },
      "scientific_discovery": {
       "accuracy": 0.5714285714285714,
       "cc": 9.066174986686397,
       "cc_by_type": {
        "choice": 63.190184049079754,
        "noul": -60,
        "score": 24.00834091097944
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 9.066174986686397
      },
      "search_retrieval": {
       "accuracy": 0.8059701492537313,
       "cc": 72.16082795188898,
       "cc_by_type": {
        "choice": 90.55118110236221,
        "noul": 42.85714285714286,
        "score": 83.07415989616185
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 72.16082795188898
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.37663636913295,
      89.02774966584359
     ],
     "I": [
      30.306448060955347,
      36.92903267922574
     ],
     "capability": [
      57.10786115237533,
      62.221081391106196
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.50529260806339,
     "capability": 61.21157187640956,
     "intelligence": 33.91785114475573
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6693333333333333,
      "calibration": 90.81218530312208,
      "cc": 54.89587859069825,
      "cc_by_type": {
       "choice": 54.89587859069825
      },
      "cc_tiered": 60.30231858733915,
      "cc_tiered_open": 67.99374492407335,
      "cc_tiered_sealed": 52.61089225060495,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.89587859069825
     },
     "noul": {
      "accuracy": 0.44266666666666665,
      "calibration": 87.7493643312945,
      "cc": -11.466666666666669,
      "cc_by_type": {
       "noul": -11.466666666666669
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.592,
      "calibration": 86.95432818977358,
      "cc": 43.697546320368,
      "cc_by_type": {
       "score": 43.697546320368
      },
      "cc_tiered": 41.451234846928045,
      "cc_tiered_open": 46.91903832814022,
      "cc_tiered_sealed": 35.983431365715866,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.697546320368
     }
    },
    "raw_on_A": {
     "calibration": 86.00622278834905,
     "capability": 59.31900660838234,
     "intelligence": 32.63179042841563
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.005333333333333301,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 27.036193256385484,
    "Emphasis on Accuracy 60:20:20": 20.91153776821196,
    "Emphasis on Cost 20:20:60": 31.50005999320053,
    "Emphasis on Speed 20:60:20": 31.851287663931,
    "Intelligence only": 15.607918222378935
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Ninh (ninhnv / xoai)",
   "axes": {
    "calibration": 58.650234835105636,
    "cost": 64.94195253546884,
    "intelligence": 27.33769023776734,
    "speed": 64.6591918236347
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04707418051854133,
      "ece_score": 90.58516389629173,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 56.76670270834282,
      "score_before_time_limit": 90.58516389629173,
      "time_limit_failed_share": 0.37333333333333335
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.08477561036707809,
      "ece": 0.03417779219939911,
      "n": 375,
      "score": 68.81746749907545,
      "score_before_time_limit": 93.16444156012018,
      "time_limit_failed_share": 0.2613333333333333
     },
     "score": {
      "interval90_coverage": 0.9146666666666666,
      "interval90_width": 0.18444444444444447,
      "n": 375,
      "nrps": 0.05480603059927663,
      "score": 50.36653429789865,
      "score_before_time_limit": 88.67347587658213,
      "time_limit_failed_share": 0.432,
      "top_ece": 0.08586222593454032
     }
    },
    "score": 58.650234835105636
   },
   "capability": 42.993962536436484,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 714,121 input + 258,678 output tokens over 1,485 decisions = 655.083501684 tokens/decision. GPU-time cost USD 0.0147435335417599/1,000; old list-price cost USD 0.050173569023569/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0147435335417599/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05329780000000001
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014743533541759862
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 540,
    "n_over_limit_answered_ok": 540,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Chon-1.0-12B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-19",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Verda GPU VM (RTX 6000 Ada), offline read-only container (--network none); author's chon.serve unchanged (xoai/chon @09ab5fa4) with the adapter's chon_config.json (bf16 LoRA r64 on the pinned google/gemma-4-12B-it @707f0a3b, letter readout, per-type calibration, yes/no commit rule, think-when-unsure pass up to 512 tokens by the base model for at most 45 % of questions) on the documented stack torch 2.9.1+cu128 / transformers 5.20.1 / peft 0.18.1",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 32.05725156238466,
    "I_sealed": 22.618128913150024,
    "base": 27.33769023776734,
    "excess": -0.4004041211210776,
    "gap": 9.439122649234633,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 50.8963923683134,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 45.265806857502355,
       "judge": 44.62755438365195,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 45.27536231884058,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -0.7618837174541815,
       "hard": -34.17965289489457,
       "judge": -54.65475090077187,
       "standard": -32.428756510153846
      }
     },
     "sealed|choice": {
      "cc": 42.8028155622999,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 63.78653113087674,
       "hard": 34.94853706487798,
       "judge": 43.58472086190012,
       "standard": 46.84665682345498
      }
     },
     "sealed|noul": {
      "cc": 25.05157117715018,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 48.27586206896552,
       "hard": 16.36363636363636,
       "judge": 10.769230769230775,
       "standard": 52.23880597014925
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": -18.093529139951947,
       "hard": -63.26646179684954,
       "judge": -58.618791804360825,
       "standard": -38.47436169952119
      }
     }
    }
   },
   "jevbench_score": 14.152024957404182,
   "key": "chon-1.0-12b",
   "last_measured_on": "2026-10-19",
   "licence": "Apache-2.0 (adapter); base google/gemma-4-12B-it Apache-2.0; serving package xoai/chon Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.32",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "ninhnv/chon-1.0-12b@cf3c0d3d5aa3230cbd36959cc4ed257b41000c5e",
   "noul_decisive": {
    "acc_among_decisive": 0.9025270758122743,
    "decisive": 277,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 277
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.80769377766468,
     "cost": 48.98575014908867,
     "intelligence": 79.86402006880415,
     "speed": 65.53984816070655
    },
    "board_release": "v1.6.43",
    "capability": 85.33585692323442,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 1,
    "capability_rank": 2,
    "composite_ci95": {
     "A": [
      63.87535788763423,
      65.45251851082296
     ],
     "B": [
      65.55109890484802,
      67.8391153066576
     ],
     "C": [
      63.87535788763423,
      65.45251851082296
     ]
    },
    "jevbench_score": 64.84806528194059,
    "measured_in": "v1.6.32",
    "method": "v1.6",
    "open_board_rank": 28,
    "rank": 49,
    "ranks": {
     "A": 49,
     "B": 34,
     "C": 36,
     "capability": 7
    },
    "scores": {
     "A": 64.84806528194059,
     "B": 66.90950425070145,
     "C": 64.84806528194059
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4174180539997906,
     "p50_s_raw": 0.1337090269998953,
     "p95_s_adjusted": 66.9027477348007,
     "p95_s_raw": 33.376373867400346
    },
    "views": {
     "Balanced 33:33:33": 59.74950639807396,
     "Emphasis on Accuracy 60:20:20": 65.5308560631613,
     "Emphasis on Cost 20:20:60": 53.91067018855768,
     "Emphasis on Speed 20:60:20": 60.97401349973243,
     "Intelligence only": 76.65679978603575
    }
   },
   "rank": 209,
   "ranked": true,
   "ranks": {
    "A": 209,
    "B": 208,
    "C": 209,
    "capability": 237
   },
   "repo": "https://huggingface.co/ninhnv/chon-1.0-12b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 14.152024957404182,
    "B": 12.345395086864695,
    "C": 9.82779510930846
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5102759639998112,
    "p50_s_raw": 0.1801379819999056,
    "p95_s_adjusted": 67.0310011336996,
    "p95_s_raw": 33.4405005668498
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": -36.7117256790645,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": -285.1351770371935
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.45544554455445546,
       "cc": -6.534473748024702,
       "cc_by_type": {
        "choice": 17.285259809119836,
        "noul": 43.99999999999999,
        "score": -80.88868105319393
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.4752475247524752,
       "cc": -39.39465509734739,
       "cc_by_type": {
        "choice": 60.975609756097555,
        "noul": -55.55555555555556,
        "score": -123.60401949258417
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 77.75646179758371,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 33.26938539275112
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 77.75646179758371
      },
      "jaggedness": {
       "accuracy": 0.3055555555555556,
       "cc": -40.08141520361054,
       "cc_by_type": {
        "choice": 6.15976900866219,
        "noul": -8.108108108108103,
        "score": -118.29590651138572
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.6996805111821086,
       "cc": 43.778009451448945,
       "cc_by_type": {
        "choice": 55.622083296645236,
        "noul": 71.13402061855669,
        "score": 4.577924439144898
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 43.778009451448945
      },
      "long_state": {
       "accuracy": 0.06666666666666667,
       "cc": -104.16096967236842,
       "cc_by_type": {
        "choice": -16.129032258064516,
        "noul": -100,
        "score": -196.3538767590408
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -30.212519107355362,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": 0,
        "score": -91.66848515711763
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.64,
       "cc": 30.49680134970566,
       "cc_by_type": {
        "choice": 53.04858167590479,
        "noul": 47.22222222222223,
        "score": -8.780399849010045
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 30.49680134970566
      },
      "negation_pair_gen": {
       "accuracy": 0.4069767441860465,
       "cc": -70.10562596759534,
       "cc_by_type": {
        "choice": 43.33333333333332,
        "noul": -46.666666666666664,
        "score": -206.9835445694527
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.625,
       "cc": 52.06391478029295,
       "cc_by_type": {
        "choice": 52.06391478029295
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 52.06391478029295
      },
      "perturb_pair": {
       "accuracy": 0.734375,
       "cc": 64.30687421060522,
       "cc_by_type": {
        "choice": 60.96096096096097,
        "noul": 100,
        "score": 31.9596616708547
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 64.30687421060522
      },
      "usecase": {
       "accuracy": 0.55,
       "cc": 15.990260307696246,
       "cc_by_type": {
        "choice": 48.529411764705905,
        "noul": 33.33333333333333,
        "score": -33.89196417495049
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 15.990260307696246
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": -44.477995629455364,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50,
        "score": -188.19589165027085
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 0.04504707569335409,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": -91.72353772837307
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0.04504707569335409
      },
      "de": {
       "accuracy": 0.75,
       "cc": 50.00742718629285,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100,
        "score": -10.154709591563925
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 50.00742718629285
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 44.241783207348895,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": 0,
        "score": 99.76445576729809
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.241783207348895
      },
      "en": {
       "accuracy": 0.5438748913987836,
       "cc": 5.935837840547715,
       "cc_by_type": {
        "choice": 43.3250753590998,
        "noul": 26.618705035971214,
        "score": -52.13626687342787
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 5.935837840547715
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 33.860230145759715,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75,
        "score": -63.63138134086115
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 33.860230145759715
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 44.86609847866006,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 99.62561784035175
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 44.86609847866006
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 62.13995588663956,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 49.94927942462455
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 62.13995588663956
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 28.235116753398046,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 50,
        "score": -15.015320130867305
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 28.235116753398046
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.6666666666666666,
       "cc": 24.05542687542328,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": -99.77448320849243
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 24.05542687542328
      },
      "ja": {
       "accuracy": 0.6,
       "cc": -19.39457564965358,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 60.00000000000001,
        "score": -175.29254347000918
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -15.954208575210638,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": -47.862625725631915
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.67239947898015,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.68386510360713
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.67239947898015
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -21.953877958184364,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": -80.58024859316781
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 35.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 35.483870967741936
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 62.28121798567983,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 80,
        "score": 29.695960347221483
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 62.28121798567983
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 28.62182908520281,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": -61.09974550809223
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 28.62182908520281
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": -18.865248226950353,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": -100
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": -21.306872210115102,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": 50,
        "score": -126.96409489121487
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 34.37659706332849,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": 100,
        "score": -0.09601526162743568
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 34.37659706332849
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -28.461345098112986,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -101.20324433388697
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5773333333333334,
       "cc": 45.40653284601696,
       "cc_by_type": {
        "choice": 45.40653284601696
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 45.40653284601696
      },
      "noul": {
       "accuracy": 0.6666666666666666,
       "cc": 33.33333333333333,
       "cc_by_type": {
        "noul": 33.33333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 33.33333333333333
      },
      "score": {
       "accuracy": 0.43466666666666665,
       "cc": -46.983912693513005,
       "cc_by_type": {
        "score": -46.983912693513005
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -33.51713526605247,
       "cc_by_type": {
        "choice": 4,
        "noul": -75,
        "score": -29.551405798157404
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 23.602515742084368,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 100,
        "score": -73.29183165573447
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 23.602515742084368
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": -17.11044795470467,
       "cc_by_type": {
        "choice": 18.699186991869915,
        "noul": 50,
        "score": -120.03053085598393
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.11764705882352941,
       "cc": -91.76413255360625,
       "cc_by_type": {
        "choice": -16.959064327485383,
        "noul": -33.333333333333336,
        "score": -225
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": -4.135618112106002,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": -35.59526013341945
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -132.30072112549024,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 66.66666666666667,
        "score": -500.6757482821311
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 10.927112300993505,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": 14.28571428571428,
        "score": -32.48476953959651
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 10.927112300993505
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -59.36527554403482,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100,
        "score": -104.92509492478739
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 30.276690027327678,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": 19.999999999999996,
        "score": 74.99673674864971
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 30.276690027327678
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": -11.981853759739328,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 33.33333333333333,
        "score": -79.3431344412451
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 38.2483397949752,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": -29.182629193885766
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 38.2483397949752
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -34.214963590681506,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -139.20403055699074
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 59.95162000305285,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 19.854860009158525
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.95162000305285
      },
      "model_routing": {
       "accuracy": 0.7741935483870968,
       "cc": 47.412797926009404,
       "cc_by_type": {
        "choice": 75.45454545454547,
        "noul": 93.93939393939394,
        "score": -27.1555456159112
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 47.412797926009404
      },
      "moderation_trust_safety": {
       "accuracy": 0.4444444444444444,
       "cc": -37.366321374133996,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100,
        "score": -54.517390226048825
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 34.71062516102369,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100,
        "score": -79.15503259492336
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 34.71062516102369
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -30.2229358544971,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50,
        "score": -81.55550707088047
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": -13.89812014922775,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": 0,
        "score": -54.73783870855282
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 88.31407046530585,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 75.61836797954744
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 88.31407046530585
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      55.21584329874216,
      60.88788711420117
     ],
     "I": [
      23.051062243612662,
      31.745239730528457
     ],
     "capability": [
      39.744609791998755,
      45.82160099193017
     ],
     "composite": null,
     "seed": 25
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 58.650234835105636,
     "capability": 42.993962536436484,
     "intelligence": 27.33769023776734
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5773333333333334,
      "calibration": 56.76670270834282,
      "cc": 45.40653284601696,
      "cc_by_type": {
       "choice": 45.40653284601696
      },
      "cc_tiered": 46.84960396530665,
      "cc_tiered_open": 50.8963923683134,
      "cc_tiered_sealed": 42.8028155622999,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 45.40653284601696
     },
     "noul": {
      "accuracy": 0.6666666666666666,
      "calibration": 68.81746749907545,
      "cc": 33.33333333333333,
      "cc_by_type": {
       "noul": 33.33333333333333
      },
      "cc_tiered": 35.16346674799538,
      "cc_tiered_open": 45.27536231884058,
      "cc_tiered_sealed": 25.05157117715018,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 33.33333333333333
     },
     "score": {
      "accuracy": 0.43466666666666665,
      "calibration": 50.36653429789865,
      "cc": -46.983912693513005,
      "cc_by_type": {
       "score": -46.983912693513005
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 57.974564399987855,
     "capability": 42.74401838693816,
     "intelligence": 27.51347237388846
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.3733333333333333,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.2613333333333333,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.43200000000000005,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 13.297319174489214,
    "Emphasis on Accuracy 60:20:20": 10.630655433758264,
    "Emphasis on Cost 20:20:60": 15.214713305437046,
    "Emphasis on Speed 20:60:20": 15.193884114609984,
    "Intelligence only": 8.172321547532945
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Ninh (ninhnv / xoai)",
   "axes": {
    "calibration": 79.86233614365636,
    "cost": 69.09820245715657,
    "intelligence": 60.27202384762607,
    "speed": 83.73773084289927
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09529694317475611,
      "ece_score": 80.94061136504878,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 80.94061136504878,
      "score_before_time_limit": 80.94061136504878,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.16186234616115597,
      "ece": 0.13262937046790704,
      "n": 375,
      "score": 73.47412590641859,
      "score_before_time_limit": 73.47412590641859,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.912,
      "interval90_width": 0.284,
      "n": 375,
      "nrps": 0.08575379234298262,
      "score": 85.17227115950172,
      "score_before_time_limit": 85.17227115950172,
      "time_limit_failed_share": 0,
      "top_ece": 0.10540039223349143
     }
    },
    "score": 79.86233614365636
   },
   "capability": 70.06717999564123,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 707,101 input + 0 output tokens over 1,485 decisions = 476.162289562 tokens/decision. GPU-time cost USD 0.0107166714921705/1,000; old list-price cost USD 0.0238081144781145/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0107166714921705/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.026628033333333335
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01071667149217047
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Chon-1.0-12B-Fast",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-25",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.1",
    "results_sha256": "9315fdbe49fce8073b9b356c4e04fafbd4778396841a88ffab1ad0f3bdf67113",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda RTX A6000 48 GB, FIN-01 (Finland, EU), non-preemptible single-GPU pod, 25 Oct 2026 00:24:58-00:49:47Z. Native author's chon.serve at 127.0.0.1:8000 with explicit local base and adapter, empty credential environment, read-only pinned mounts, --network none and host egress cut proven before sealed input; no vLLM or think pass. Input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b. Host validator PASS, 1500/1500 native HTTP 200; mean latency 0.219292 s. Node shred (0 files left) and exact instance and volume teardown verified. About USD 0.41 rental of USD 0.98 booking is separate from estimated inference tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 65.94466596827355,
    "I_sealed": 54.59938172697859,
    "base": 60.27202384762607,
    "excess": 1.5057574709392512,
    "gap": 11.345284241294962,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 68.03622221108897,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 67.9144385026738,
       "judge": 56.49307844429796,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 68.67632850241546,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 60.00000000000001,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 61.12144719131625,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 59.88341965855466,
       "hard": 52.547981638992034,
       "judge": 63.697518343228964,
       "standard": 75.02328533447643
      }
     },
     "sealed|choice": {
      "cc": 70.36502476739707,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 67.2140626806985,
       "judge": 70.91087169441724,
       "standard": 74.3724952541658
      }
     },
     "sealed|noul": {
      "cc": 47.98061537021907,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 62.06896551724137,
       "hard": 43.636363636363626,
       "judge": 32.30769230769231,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 45.45250504331964,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 49.868601481636865,
       "hard": 46.974661913197494,
       "judge": 46.02724322515706,
       "standard": 39.33803581164918
      }
     }
    }
   },
   "jevbench_score": 72.0367713753772,
   "key": "chon-1.0-12b-fast",
   "last_measured_on": "2026-10-25",
   "licence": "Apache-2.0 (adapter); base google/gemma-4-12B-it Apache-2.0; serving package xoai/chon Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.7.1",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "ninhnv/chon-1.0-12b-fast@92c960ef1245f604642538c562ed9dec89bab48f",
   "new_in": "v1.7.1",
   "noul_decisive": {
    "acc_among_decisive": 0.776,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 56,
   "ranked": true,
   "ranks": {
    "A": 56,
    "B": 59,
    "C": 44,
    "capability": 100
   },
   "repo": "https://huggingface.co/ninhnv/chon-1.0-12b-fast",
   "scores": {
    "A": 72.0367713753772,
    "B": 69.33020119237318,
    "C": 72.0367713753772
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4998688179997771,
    "p50_s_raw": 0.17493440899988855,
    "p95_s_adjusted": 0.8460009866999939,
    "p95_s_raw": 0.34800049334999694
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author's own xoai/chon v1.1.0 at 3fc64c0aa3b02cf234acc16c397bd2ce70b34df4; independently source-reviewed before execution; the Fast code gate and pre-registration amendments 1-2 apply. Adapter ninhnv/chon-1.0-12b-fast at 92c960ef1245f604642538c562ed9dec89bab48f, base google/gemma-4-12B-it at 707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7 (evaluator-selected; training revision unproven). Python 3.12, torch 2.9.1+cu128, transformers 5.20.2, peft 0.18.1, BF16 CUDA SDPA, default adapter merge/unload, no vLLM and no think pass. Image sha256:4659ede5b4a66983cb113724bb5eac662f2fa1f2ec9bb11fbb2a9e06dcb77751. Dependency lock identical to attempt 2 except transitive starlette 1.9.0 to 1.10.0. Frozen run_v15.py and typesafe adapter unchanged. Same author, base and identical adapter weight/calibration as the Chon-1.0-12B family; Fast disables thinking. PEFT LoRA r64, native choice/noul/score letter readout and author calibration: temperatures choice 1.0181, noul 0.3915, score 0.7489; post-temperatures 0.8/2.5/1.0; generic noul commit band [0.199,0.801]. Submitted max_state_tokens=6000 and truncate=true, silently retaining beginning plus omission marker for ordinary text/JSON, recent conversation turns or tail of a single huge last message; state limit excludes prompt scaffold. Counts from the sealed inference: 15 truncated states, all answered; 0 grouped (>16-option) decisions. Zero output tokens on every answer, no think pass, no vLLM. 1500/1500 native HTTP 200, zero refusals, no dropped/repaired/retried items. One self-hosted use of v1.6-regular-20261012 under a zero-refusal custody policy; the second and last admitted Chon-12B family use of this draw. ATTEMPT HISTORY: two initial image-create HTTP 400 refusals created no instance (Amendment 1). Actual attempt 1 on FIN-02 never obtained SSH; attempt 2 on FIN-01 stopped before sealed input because compose_proof.py shadowed hostcut() (UnboundLocalError); both stopped before any sealed byte. Amendment 2 admits only the host-cut variable fix and a third attempt, and raises the run envelope from USD 2 to USD 3 (owner-approved), with no ledger reset. Attempt 3 is this successful measurement. Previous attempts are receipts only, never scored or merged.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 55.61386026706186,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": -24.82508586548109
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 55.61386026706186
      },
      "calibration_gen": {
       "accuracy": 0.6534653465346535,
       "cc": 46.77269677417467,
       "cc_by_type": {
        "choice": 45.91728525980912,
        "noul": 52,
        "score": 42.40080506271489
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 46.77269677417467
      },
      "composite_gen": {
       "accuracy": 0.6534653465346535,
       "cc": 33.50080914633481,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -18.518518518518523,
        "score": 51.54127116077498
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 33.50080914633481
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 79.84416796436119,
       "cc_by_type": {
        "choice": 71.42857142857143,
        "noul": 100,
        "score": 68.10393246451214
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 79.84416796436119
      },
      "jaggedness": {
       "accuracy": 0.5722222222222222,
       "cc": 34.82107222942229,
       "cc_by_type": {
        "choice": 48.0269489894129,
        "noul": 35.13513513513513,
        "score": 21.301132563718838
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 34.82107222942229
      },
      "llm_router": {
       "accuracy": 0.8434504792332268,
       "cc": 72.19829138635906,
       "cc_by_type": {
        "choice": 80.98089284141939,
        "noul": 77.31958762886597,
        "score": 58.294393688791814
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 72.19829138635906
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": 1.7754623463288652,
       "cc_by_type": {
        "choice": 22.58064516129032,
        "noul": 0,
        "score": -17.254258122303725
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 1.7754623463288652
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 39.57161744961175,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 50,
        "score": 5.8282544106909135
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 39.57161744961175
      },
      "multilingual": {
       "accuracy": 0.7454545454545455,
       "cc": 64.34910879335062,
       "cc_by_type": {
        "choice": 68.6990544506032,
        "noul": 66.66666666666667,
        "score": 57.681605262781964
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 64.34910879335062
      },
      "negation_pair_gen": {
       "accuracy": 0.6395348837209303,
       "cc": 28.158827836099316,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": -6.666666666666665,
        "score": 24.476483508297942
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 28.158827836099316
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 76.57390746154535,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100,
        "score": 56.74874941166308
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 76.57390746154535
      },
      "usecase": {
       "accuracy": 0.7545454545454545,
       "cc": 66.30050975674965,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 66.66666666666667,
        "score": 53.06819593691561
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 66.30050975674965
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 31.258474005637225,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100,
        "score": -49.08172084023117
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 31.258474005637225
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 60.936698863370204,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 49.47676325677729
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 60.936698863370204
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 87.80203381651167,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 87.29990675926953
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.80203381651167
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 44.21702071538203,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": 0,
        "score": 99.69016829139747
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.21702071538203
      },
      "en": {
       "accuracy": 0.7280625543006082,
       "cc": 55.680867906218964,
       "cc_by_type": {
        "choice": 70.34451617627316,
        "noul": 49.64028776978417,
        "score": 47.057799772599566
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 55.680867906218964
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 71.26492940300356,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 23.582716430870367
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.26492940300356
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 67.07735405404276,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 100,
        "score": 99.59271789983318
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 67.07735405404276
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 84.00310107254595,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 94.36224439410843
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.00310107254595
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 70.81204102891128,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 75,
        "score": 37.43612308673385
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 70.81204102891128
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.6666666666666666,
       "cc": 48.78689779524279,
       "cc_by_type": {
        "choice": 57.911145752143405,
        "noul": 100,
        "score": -11.550452366415032
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 48.78689779524279
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 48.84815223073597,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 80,
        "score": -4.8614209884910276
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 48.84815223073597
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 40.18646289538529,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 20.559388686155867
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 40.18646289538529
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.90911259411952,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.72733778235856
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.90911259411952
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 54.15646195885322,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 33.33333333333333,
        "score": 55.11007851725233
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 54.15646195885322
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 77.69540162059563,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80,
        "score": 60.70364032518092
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 77.69540162059563
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 75.25127494799914,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 57.574685185777
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 75.25127494799914
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 47.09279155367265,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 19.999999999999996,
        "score": 72.342204448252
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 47.09279155367265
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": 30.426691925695735,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50,
        "score": 6.497467081435026
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 30.426691925695735
      },
      "uk": {
       "accuracy": 1,
       "cc": 93.54379283510507,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 80.63137850531521
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 93.54379283510507
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 48.08248731888978,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": 36.33785743689535
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 48.08248731888978
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7706666666666667,
       "cc": 70.37830804263382,
       "cc_by_type": {
        "choice": 70.37830804263382
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 70.37830804263382
      },
      "noul": {
       "accuracy": 0.776,
       "cc": 55.2,
       "cc_by_type": {
        "noul": 55.2
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 55.2
      },
      "score": {
       "accuracy": 0.6213333333333333,
       "cc": 48.093629312192746,
       "cc_by_type": {
        "score": 48.093629312192746
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 48.093629312192746
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 44.011431488992365,
       "cc_by_type": {
        "choice": 68,
        "noul": 25,
        "score": 39.03429446697708
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 44.011431488992365
      },
      "code_linting": {
       "accuracy": 0.7083333333333334,
       "cc": 65.87524630896445,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 16.259279299564167
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 65.87524630896445
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 47.297043205105886,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 24.411454818569688
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 47.297043205105886
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 1.7462718746566361,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -33.333333333333336,
        "score": -2.9483188789540593
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 1.7462718746566361
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 27.49179181034607,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 44.79421601074834
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 27.49179181034607
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": 30.762570065204837,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 100,
        "score": -57.39782439558046
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 30.762570065204837
      },
      "financial_crime": {
       "accuracy": 0.8666666666666667,
       "cc": 79.25648955699079,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 71.42857142857142,
        "score": 66.34089724240093
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 79.25648955699079
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 33.906002663543156,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 0,
        "score": 43.52985468052495
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 33.906002663543156
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 56.86715668674676,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 73.10147006024027
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 56.86715668674676
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 42.18255394658329,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 66.66666666666667,
        "score": 36.96878960563137
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 42.18255394658329
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 58.39111756722594,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 31.24570412286647
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 58.39111756722594
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 31.9937049146201,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 33.33333333333333,
        "score": -5.094154073343993
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 31.9937049146201
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 75.24193774811961,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 65.72581324435882
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 75.24193774811961
      },
      "model_routing": {
       "accuracy": 0.880184331797235,
       "cc": 76.41756437844484,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 93.93939393939394,
        "score": 44.66394854658991
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 76.41756437844484
      },
      "moderation_trust_safety": {
       "accuracy": 0.8888888888888888,
       "cc": 54.83734891908859,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0,
        "score": 76.02836153653641
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 54.83734891908859
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 78.6309973182117,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100,
        "score": 52.60608387664063
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 78.6309973182117
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 57.640022376607085,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50,
        "score": 52.47671737612667
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 57.640022376607085
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 44.74052674831635,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 44.36650778118094
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 44.74052674831635
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 94.12361473967924,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 93.04700080266764
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.12361473967924
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.51196354025824,
      83.11551724558446
     ],
     "I": [
      55.75998806767451,
      64.1152608498756
     ],
     "capability": [
      66.25473557801327,
      73.131897517281
     ],
     "composite": null,
     "seed": 26
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.86233614365636,
     "capability": 70.06717999564123,
     "intelligence": 60.27202384762607
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7706666666666667,
      "calibration": 80.94061136504878,
      "cc": 70.37830804263382,
      "cc_by_type": {
       "choice": 70.37830804263382
      },
      "cc_tiered": 69.20062348924301,
      "cc_tiered_open": 68.03622221108897,
      "cc_tiered_sealed": 70.36502476739707,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.37830804263382
     },
     "noul": {
      "accuracy": 0.776,
      "calibration": 73.47412590641859,
      "cc": 55.2,
      "cc_by_type": {
       "noul": 55.2
      },
      "cc_tiered": 58.328471936317264,
      "cc_tiered_open": 68.67632850241546,
      "cc_tiered_sealed": 47.98061537021907,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 55.2
     },
     "score": {
      "accuracy": 0.6213333333333333,
      "calibration": 85.17227115950172,
      "cc": 48.093629312192746,
      "cc_by_type": {
       "score": 48.093629312192746
      },
      "cc_tiered": 53.28697611731795,
      "cc_tiered_open": 61.12144719131625,
      "cc_tiered_sealed": 45.45250504331964,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 48.093629312192746
     }
    },
    "raw_on_A": {
     "calibration": 82.4660233630633,
     "capability": 70.99383666874715,
     "intelligence": 59.521649974431
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.75827593431589,
    "Emphasis on Accuracy 60:20:20": 65.62666683625854,
    "Emphasis on Cost 20:20:60": 69.49273898990214,
    "Emphasis on Speed 20:60:20": 74.74987274468121,
    "Intelligence only": 60.27202384762608
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Ninh (ninhnv / xoai)",
   "axes": {
    "calibration": 81.517233540399,
    "cost": 83.52175108923268,
    "intelligence": 55.488532895897606,
    "speed": 88.71017618901561
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.100828687812117,
      "ece_score": 79.8342624375766,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 79.8342624375766,
      "score_before_time_limit": 79.8342624375766,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.14804321133236406,
      "ece": 0.09021893668291027,
      "n": 375,
      "score": 81.95621266341794,
      "score_before_time_limit": 81.95621266341794,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8906666666666667,
      "interval90_width": 0.37333333333333346,
      "n": 375,
      "nrps": 0.1068733141892066,
      "score": 82.76122552020249,
      "score_before_time_limit": 82.76122552020249,
      "time_limit_failed_share": 0,
      "top_ece": 0.11895108770337182
     }
    },
    "score": 81.517233540399
   },
   "capability": 68.5028832181483,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 4B active parameters; throughput 80585.75 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 701,161 input + 0 output tokens over 1,485 decisions = 472.162289562 tokens/decision. GPU-time cost USD 0.00354221537428728/1,000; old list-price cost USD 0.00944324579124579/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00354221537428728/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.010571213333333334
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003542215374287284
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Chon-1.0-E4B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-25",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.3",
    "results_sha256": "171680e9f2f7b884b7a206e15d280bdfc189c2b35ad9c1f578709902c59a6892",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda RTX A6000 48 GB, FIN-01 (Finland, EU), non-preemptible, measured 25 Oct 2026. Disposable read-only container with no network; network cut proven inside the container before sealed input. Author's chon.serve 1.0.0 at revision 35fb1142d9f4e68850bcb12d2df8b0b3564e0fc0, --device cuda --dtype bfloat16 --no-think; Runtime pinned by the build: Python 3.11 container (author documents Python 3.12), PyTorch 2.9.1+cu128, transformers 5.19.0, peft 0.18.1 and huggingface-hub 1.33.0; no runtime version readback was collected. GPU sampler CSV was empty; host CPU model was not retained in the collected receipts. Published Speed applies the standard self-hosted x2 + 0.15 s adjustment (assumption, not measured). Rental about USD 0.14 is separate from the unchanged base-model inference cost estimate.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 60.22741076368507,
    "I_sealed": 50.74965502811013,
    "base": 55.488532895897606,
    "excess": -0.361771034780773,
    "gap": 9.477755735574938,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 62.113835115843074,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 58.47750865051904,
       "judge": 64.40342781806197,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 74.85507246376812,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 50,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 43.71332471144403,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.93899636921513,
       "hard": 37.56822962376864,
       "judge": 57.10197521858322,
       "standard": 30.307703297200483
      }
     },
     "sealed|choice": {
      "cc": 62.34570330349263,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 54.72418179715507,
       "judge": 62.09598432908915,
       "standard": 71.52499472685088
      }
     },
     "sealed|noul": {
      "cc": 56.03664580401585,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 48.27586206896552,
       "hard": 56.363636363636374,
       "judge": 50.76923076923077,
       "standard": 67.16417910447761
      }
     },
     "sealed|score": {
      "cc": 33.86661597682193,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 43.48214824219369,
       "hard": 32.242756280693605,
       "judge": 33.31204608863996,
       "standard": 33.13842406866567
      }
     }
    }
   },
   "jevbench_score": 74.71792231346033,
   "key": "chon-1.0-e4b",
   "last_measured_on": "2026-10-25",
   "licence": "Apache-2.0 (adapter card and base licence agree)",
   "listing": "ranked",
   "measured_in": "v1.7.3",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "ninhnv/chon-1.0-e4b@b7b9ecdaa25d83696fb1c82c3b0acaff2923137a",
   "noul_decisive": {
    "acc_among_decisive": 0.8,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "addendum": null,
    "api_flag": false,
    "api_subset": false,
    "author": "Ninh (ninhnv / xoai)",
    "axes": {
     "calibration": 81.82393852673533,
     "cost": 70.74636118712607,
     "intelligence": 54.96148352069754,
     "speed": 43.95627533510597
    },
    "board_release": "v1.7.2",
    "calibration": {
     "parts": {
      "choice": {
       "ece_hard": 0.09101480224243556,
       "ece_score": 81.79703955151288,
       "mean_tvd": null,
       "n": 750,
       "n_ece_hard": 315,
       "n_tvd": 0,
       "score": 81.79703955151288
      },
      "noul": {
       "abstention_rate": 0,
       "brier": 0.15139765553945303,
       "ece": 0.09557442624031161,
       "n": 375,
       "score": 80.88511475193768
      },
      "score": {
       "interval90_coverage": 0.8906666666666667,
       "interval90_width": 0.3764444444444446,
       "n": 375,
       "nrps": 0.10648183537819128,
       "score": 82.78966127675545,
       "top_ece": 0.1188624695433499
      }
     },
     "score": 81.82393852673533
    },
    "capability": 68.39271102371643,
    "composite_ci95": {
     "A": [
      44.392703292458805,
      46.89203249517508
     ],
     "B": [
      43.17507195851998,
      46.534331841823644
     ],
     "C": [
      31.09870948973084,
      44.91059204066695
     ]
    },
    "cost": {
     "basis": "Exact listed rate of the base model google/gemma-4-E4B-it (DeepInfra, frozen 25 Sep 2026 catalog snapshot, USD 0.02/M input, 0.10/M output); this line is added to the frozen pricing module at its next re-freeze. No output tokens under the author's --no-think mode. COMMON COST BASIS: the published cost uses the same remaining 1,485 items as every other row: 701,161 input tokens, USD 0.009443245791245792 per 1,000 decisions (0.146x cap). ALL-ITEMS DIAGNOSTIC: USD 0.010571213333333333 per 1,000 over all 1,500 items (0.164x cap). All items still count for Intelligence.",
     "common_basis": {
      "n_items": 1485,
      "rule": "common",
      "usd_per_1000_all_items": 0.010571213333333334
     },
     "kind": "listed",
     "usage_estimated_rows": 0,
     "usd_per_1000": 0.009443245791245792
    },
    "decision_limit": null,
    "display": "Chon-1.0-E4B",
    "draw": {
     "drawn_on": "2026-10-12",
     "g_med": 9.839526770355711,
     "id": "v1.6-regular-20261012",
     "label": "Second regular draw",
     "measured_on": "2026-10-19",
     "offset": {
      "C": 0,
      "I": 0
     },
     "offset_ci95": null,
     "release": "v1.6.43",
     "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
     "status": "anchor-runs-pending"
    },
    "endpoint_condition": "Our own CPU server (AMD EPYC, 32 cores), disposable container with no network; the network cut was proven inside the container before it read any sealed item. Served by the author's own chon.serve 1.0.0 in its documented CPU mode (--device cpu --no-think). Published Speed applies the standard self-hosted x2 + 0.15 s adjustment (assumption, not measured).",
    "endpoint_kind": "cpu",
    "full_coverage": true,
    "intelligence": {
     "I_open": 59.11785286864142,
     "I_sealed": 50.805114172753655,
     "base": 54.96148352069754,
     "excess": -1.5267880744679445,
     "gap": 8.312738695887766,
     "penalty": 1,
     "per_type_split": {
      "open|choice": {
       "cc": 62.113835115843074,
       "n": {
        "easy": 33,
        "hard": 68,
        "judge": 32,
        "standard": 17
       },
       "tiers": {
        "easy": 84.43243243243244,
        "hard": 58.47750865051904,
        "judge": 64.40342781806197,
        "standard": 54.79280033486815
       }
      },
      "open|noul": {
       "cc": 71.3768115942029,
       "n": {
        "easy": 14,
        "hard": 23,
        "judge": 20,
        "standard": 18
       },
       "tiers": {
        "easy": 100,
        "hard": 82.6086956521739,
        "judge": 50,
        "standard": 66.66666666666667
       }
      },
      "open|score": {
       "cc": 43.862911895878305,
       "n": {
        "easy": 9,
        "hard": 29,
        "judge": 21,
        "standard": 16
       },
       "tiers": {
        "easy": 54.71354470950146,
        "hard": 37.50543489517916,
        "judge": 57.48103056832748,
        "standard": 30.725371481791253
       }
      },
      "sealed|choice": {
       "cc": 63.02647928723724,
       "n": {
        "easy": 70,
        "hard": 247,
        "judge": 146,
        "standard": 137
       },
       "tiers": {
        "easy": 75.22236340533672,
        "hard": 55.76500520411702,
        "judge": 62.97747306562194,
        "standard": 71.52499472685088
       }
      },
      "sealed|noul": {
       "cc": 55.309373076743114,
       "n": {
        "easy": 58,
        "hard": 110,
        "judge": 65,
        "standard": 67
       },
       "tiers": {
        "easy": 48.27586206896552,
        "hard": 54.54545454545454,
        "judge": 50.76923076923077,
        "standard": 67.16417910447761
       }
      },
      "sealed|score": {
       "cc": 34.079490154280634,
       "n": {
        "easy": 35,
        "hard": 123,
        "judge": 67,
        "standard": 75
       },
       "tiers": {
        "easy": 44.05642671414762,
        "hard": 32.35747991578803,
        "judge": 33.25594585627047,
        "standard": 33.77035879834759
       }
      }
     }
    },
    "jevbench_score": 45.93476445894657,
    "key": "chon-1.0-e4b",
    "last_measured_on": "2026-10-19",
    "licence": "Apache-2.0 (adapter card and base licence agree)",
    "listing": "remeasure-pending",
    "measured_in": "v1.6.32",
    "measurement_date_status": "measured",
    "method": "v1.6",
    "model_pin": "ninhnv/chon-1.0-e4b@b7b9ecdaa25d83696fb1c82c3b0acaff2923137a",
    "not_ranked_because": "Re-measure pending under the 10 s rule: this row's per-decision times were recorded on a loaded CPU host, so they cannot be scored under the time limit. Shown with its v1.6 values until it is measured again.",
    "noul_decisive": {
     "acc_among_decisive": 0.7946666666666666,
     "decisive": 375,
     "decisive_rate": 1,
     "n": 375,
     "supported": true,
     "valid": 375
    },
    "paid_fast_lane": false,
    "planned_weight_denominator": 1,
    "rank": null,
    "ranked": false,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "repo": "https://huggingface.co/ninhnv/chon-1.0-e4b",
    "scores": {
     "A": 45.93476445894657,
     "B": 45.19903845098299,
     "C": 38.5439204552548
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 44.504827468981965,
     "p50_s_raw": 22.177413734490983,
     "p95_s_adjusted": 90.35773683558801,
     "p95_s_raw": 45.103868417794004
    },
    "status": {
     "answered_ok": 1500,
     "missing": 0,
     "rows": 1500,
     "status": "complete"
    },
    "support": {
     "choice": "native",
     "noul": "native",
     "score": "native"
    },
    "v16": {
     "api_subset": false,
     "api_subset_tag": null,
     "breakdowns": {
      "family": {
       "calibration": {
        "accuracy": 0.8636363636363636,
        "cc": 69.10800085114204,
        "cc_by_type": {
         "choice": 83.33333333333333,
         "noul": 100,
         "score": 23.99066922009281
        },
        "low_n": true,
        "n": 22,
        "n_by_type": {
         "choice": 16,
         "noul": 4,
         "score": 2
        },
        "score": 69.10800085114204
       },
       "calibration_gen": {
        "accuracy": 0.4752475247524752,
        "cc": 27.39797571103509,
        "cc_by_type": {
         "choice": 36.37327677624603,
         "noul": 43.99999999999999,
         "score": 1.8206503568592547
        },
        "low_n": false,
        "n": 101,
        "n_by_type": {
         "choice": 41,
         "noul": 25,
         "score": 35
        },
        "score": 27.39797571103509
       },
       "composite_gen": {
        "accuracy": 0.6138613861386139,
        "cc": 33.0387450611331,
        "cc_by_type": {
         "choice": 64.22764227642277,
         "noul": 3.703703703703698,
         "score": 31.184889203272835
        },
        "low_n": false,
        "n": 101,
        "n_by_type": {
         "choice": 41,
         "noul": 27,
         "score": 33
        },
        "score": 33.0387450611331
       },
       "fairness": {
        "accuracy": 0.8125,
        "cc": 75.50614497103835,
        "cc_by_type": {
         "choice": 100,
         "noul": 60.00000000000001,
         "score": 66.51843491311506
        },
        "low_n": false,
        "n": 48,
        "n_by_type": {
         "choice": 15,
         "noul": 15,
         "score": 18
        },
        "score": 75.50614497103835
       },
       "jaggedness": {
        "accuracy": 0.5444444444444444,
        "cc": 28.31724015648845,
        "cc_by_type": {
         "choice": 49.4706448508181,
         "noul": 35.13513513513513,
         "score": 0.34594048351211937
        },
        "low_n": false,
        "n": 180,
        "n_by_type": {
         "choice": 94,
         "noul": 37,
         "score": 49
        },
        "score": 28.31724015648845
       },
       "llm_router": {
        "accuracy": 0.7987220447284346,
        "cc": 68.96682037997233,
        "cc_by_type": {
         "choice": 65.92409967420974,
         "noul": 87.62886597938144,
         "score": 53.34749548632583
        },
        "low_n": false,
        "n": 313,
        "n_by_type": {
         "choice": 152,
         "noul": 97,
         "score": 64
        },
        "score": 68.96682037997233
       },
       "long_state": {
        "accuracy": 0.3333333333333333,
        "cc": 9.378307964358866,
        "cc_by_type": {
         "choice": 9.677419354838706,
         "noul": 0,
         "score": 18.45750453823789
        },
        "low_n": true,
        "n": 15,
        "n_by_type": {
         "choice": 10,
         "noul": 2,
         "score": 3
        },
        "score": 9.378307964358866
       },
       "ml_usecase": {
        "accuracy": 0.5185185185185185,
        "cc": 22.828236669030773,
        "cc_by_type": {
         "choice": 25.773195876288657,
         "noul": 25,
         "score": 17.711514130803664
        },
        "low_n": true,
        "n": 27,
        "n_by_type": {
         "choice": 11,
         "noul": 8,
         "score": 8
        },
        "score": 22.828236669030773
       },
       "multilingual": {
        "accuracy": 0.7636363636363637,
        "cc": 64.05836316914788,
        "cc_by_type": {
         "choice": 69.67720899902184,
         "noul": 66.66666666666667,
         "score": 55.83121384175516
        },
        "low_n": false,
        "n": 275,
        "n_by_type": {
         "choice": 134,
         "noul": 72,
         "score": 69
        },
        "score": 64.05836316914788
       },
       "negation_pair_gen": {
        "accuracy": 0.5813953488372093,
        "cc": 24.426752663997473,
        "cc_by_type": {
         "choice": 50,
         "noul": 13.33333333333333,
         "score": 9.946924658659084
        },
        "low_n": false,
        "n": 86,
        "n_by_type": {
         "choice": 40,
         "noul": 30,
         "score": 16
        },
        "score": 24.426752663997473
       },
       "option_order": {
        "accuracy": 0.8125,
        "cc": 76.03195739014647,
        "cc_by_type": {
         "choice": 76.03195739014647
        },
        "low_n": false,
        "n": 48,
        "n_by_type": {
         "choice": 48
        },
        "score": 76.03195739014647
       },
       "perturb_pair": {
        "accuracy": 0.765625,
        "cc": 69.51839721905846,
        "cc_by_type": {
         "choice": 69.96996996996997,
         "noul": 100,
         "score": 38.5852216872054
        },
        "low_n": false,
        "n": 64,
        "n_by_type": {
         "choice": 44,
         "noul": 10,
         "score": 10
        },
        "score": 69.51839721905846
       },
       "usecase": {
        "accuracy": 0.7227272727272728,
        "cc": 62.18107390251658,
        "cc_by_type": {
         "choice": 73.03921568627452,
         "noul": 70.83333333333333,
         "score": 42.6706726879419
        },
        "low_n": false,
        "n": 220,
        "n_by_type": {
         "choice": 104,
         "noul": 48,
         "score": 68
        },
        "score": 62.18107390251658
       }
      },
      "lang": {
       "ar": {
        "accuracy": 0.375,
        "cc": 25.547602788904285,
        "cc_by_type": {
         "choice": 4.76190476190476,
         "noul": 100,
         "score": -28.1190963951919
        },
        "low_n": true,
        "n": 16,
        "n_by_type": {
         "choice": 7,
         "noul": 4,
         "score": 5
        },
        "score": 25.547602788904285
       },
       "cs": {
        "accuracy": 0.8571428571428571,
        "cc": 89.47368421052632,
        "cc_by_type": {
         "choice": 78.94736842105263,
         "noul": 100
        },
        "low_n": true,
        "n": 7,
        "n_by_type": {
         "choice": 6,
         "noul": 1
        },
        "score": 89.47368421052632
       },
       "da": {
        "accuracy": 0.6153846153846154,
        "cc": 28.902401515803394,
        "cc_by_type": {
         "choice": 79.26267281105991,
         "noul": -33.333333333333336,
         "score": 40.777865069683614
        },
        "low_n": true,
        "n": 13,
        "n_by_type": {
         "choice": 6,
         "noul": 3,
         "score": 4
        },
        "score": 28.902401515803394
       },
       "de": {
        "accuracy": 0.875,
        "cc": 82.54988241544932,
        "cc_by_type": {
         "choice": 68.14159292035399,
         "noul": 100,
         "score": 79.50805432599395
        },
        "low_n": false,
        "n": 32,
        "n_by_type": {
         "choice": 16,
         "noul": 8,
         "score": 8
        },
        "score": 82.54988241544932
       },
       "el": {
        "accuracy": 0.6,
        "cc": 40.72667100867935,
        "cc_by_type": {
         "choice": 66.4804469273743,
         "noul": -33.333333333333336,
         "score": 89.03289943199708
        },
        "low_n": true,
        "n": 15,
        "n_by_type": {
         "choice": 8,
         "noul": 6,
         "score": 1
        },
        "score": 40.72667100867935
       },
       "en": {
        "accuracy": 0.6750651607298002,
        "cc": 49.948331118583404,
        "cc_by_type": {
         "choice": 62.43638715661266,
         "noul": 55.395683453237424,
         "score": 32.01292274590012
        },
        "low_n": false,
        "n": 1151,
        "n_by_type": {
         "choice": 588,
         "noul": 278,
         "score": 285
        },
        "score": 49.948331118583404
       },
       "es": {
        "accuracy": 0.8709677419354839,
        "cc": 68.23881788573766,
        "cc_by_type": {
         "choice": 100,
         "noul": 100,
         "score": 4.716453657212982
        },
        "low_n": false,
        "n": 31,
        "n_by_type": {
         "choice": 13,
         "noul": 8,
         "score": 10
        },
        "score": 68.23881788573766
       },
       "fi": {
        "accuracy": 0.75,
        "cc": 75.46287033873422,
        "cc_by_type": {
         "choice": 34.42622950819673,
         "noul": 100,
         "score": 91.9623815080059
        },
        "low_n": true,
        "n": 8,
        "n_by_type": {
         "choice": 4,
         "noul": 3,
         "score": 1
        },
        "score": 75.46287033873422
       },
       "fr": {
        "accuracy": 0.7142857142857143,
        "cc": 73.32904837911452,
        "cc_by_type": {
         "choice": 36.47058823529412,
         "noul": 100,
         "score": 83.51655690204947
        },
        "low_n": true,
        "n": 14,
        "n_by_type": {
         "choice": 6,
         "noul": 2,
         "score": 6
        },
        "score": 73.32904837911452
       },
       "hi": {
        "accuracy": 0.8571428571428571,
        "cc": 61.5339984543601,
        "cc_by_type": {
         "choice": 83.24022346368716,
         "noul": 50,
         "score": 51.36177189939314
        },
        "low_n": true,
        "n": 21,
        "n_by_type": {
         "choice": 8,
         "noul": 8,
         "score": 5
        },
        "score": 61.5339984543601
       },
       "id": {
        "accuracy": 0.75,
        "cc": 77.77777777777777,
        "cc_by_type": {
         "choice": 55.55555555555555,
         "noul": 100
        },
        "low_n": true,
        "n": 4,
        "n_by_type": {
         "choice": 3,
         "noul": 1
        },
        "score": 77.77777777777777
       },
       "it": {
        "accuracy": 0.8,
        "cc": 71.82276559964139,
        "cc_by_type": {
         "choice": 85.97038191738113,
         "noul": 100,
         "score": 29.497914881542997
        },
        "low_n": true,
        "n": 15,
        "n_by_type": {
         "choice": 9,
         "noul": 3,
         "score": 3
        },
        "score": 71.82276559964139
       },
       "ja": {
        "accuracy": 0.72,
        "cc": 44.97147892919882,
        "cc_by_type": {
         "choice": 28.514694201747414,
         "noul": 80,
         "score": 26.39974258584905
        },
        "low_n": true,
        "n": 25,
        "n_by_type": {
         "choice": 9,
         "noul": 10,
         "score": 6
        },
        "score": 44.97147892919882
       },
       "ko": {
        "accuracy": 0.7,
        "cc": 76.88577914875698,
        "cc_by_type": {
         "choice": 100,
         "noul": 100,
         "score": 30.657337446270972
        },
        "low_n": true,
        "n": 10,
        "n_by_type": {
         "choice": 3,
         "noul": 2,
         "score": 5
        },
        "score": 76.88577914875698
       },
       "mixed": {
        "accuracy": 1,
        "cc": 99.2462144174098,
        "cc_by_type": {
         "choice": 100,
         "noul": 100,
         "score": 97.73864325222938
        },
        "low_n": true,
        "n": 14,
        "n_by_type": {
         "choice": 8,
         "noul": 3,
         "score": 3
        },
        "score": 99.2462144174098
       },
       "nl": {
        "accuracy": 0.7,
        "cc": 45.84212257699509,
        "cc_by_type": {
         "choice": 74.02597402597404,
         "noul": 33.33333333333333,
         "score": 30.16706037167789
        },
        "low_n": true,
        "n": 10,
        "n_by_type": {
         "choice": 5,
         "noul": 3,
         "score": 2
        },
        "score": 45.84212257699509
       },
       "no": {
        "accuracy": 0.8333333333333334,
        "cc": 67.74193548387098,
        "cc_by_type": {
         "choice": 35.483870967741936,
         "noul": 100
        },
        "low_n": true,
        "n": 6,
        "n_by_type": {
         "choice": 2,
         "noul": 4
        },
        "score": 67.74193548387098
       },
       "pl": {
        "accuracy": 0.7948717948717948,
        "cc": 68.0579119218342,
        "cc_by_type": {
         "choice": 92.382564536606,
         "noul": 60.00000000000001,
         "score": 51.79117122889663
        },
        "low_n": false,
        "n": 39,
        "n_by_type": {
         "choice": 17,
         "noul": 10,
         "score": 12
        },
        "score": 68.0579119218342
       },
       "pt": {
        "accuracy": 0.8214285714285714,
        "cc": 75.20820126987579,
        "cc_by_type": {
         "choice": 57.57218621096051,
         "noul": 100,
         "score": 68.05241759866685
        },
        "low_n": true,
        "n": 28,
        "n_by_type": {
         "choice": 12,
         "noul": 5,
         "score": 11
        },
        "score": 75.20820126987579
       },
       "sv": {
        "accuracy": 0.5454545454545454,
        "cc": 41.70136243631472,
        "cc_by_type": {
         "choice": 23.404255319148945,
         "noul": 60.00000000000001,
         "score": 41.699831989795186
        },
        "low_n": true,
        "n": 11,
        "n_by_type": {
         "choice": 5,
         "noul": 5,
         "score": 1
        },
        "score": 41.70136243631472
       },
       "tr": {
        "accuracy": 0.6153846153846154,
        "cc": 40.435388558025224,
        "cc_by_type": {
         "choice": 56.52173913043478,
         "noul": 50,
         "score": 14.784426543640894
        },
        "low_n": true,
        "n": 13,
        "n_by_type": {
         "choice": 6,
         "noul": 4,
         "score": 3
        },
        "score": 40.435388558025224
       },
       "uk": {
        "accuracy": 1,
        "cc": 94.04904312421029,
        "cc_by_type": {
         "choice": 100,
         "noul": 100,
         "score": 82.14712937263086
        },
        "low_n": true,
        "n": 7,
        "n_by_type": {
         "choice": 4,
         "noul": 1,
         "score": 2
        },
        "score": 94.04904312421029
       },
       "zh": {
        "accuracy": 0.6,
        "cc": 23.097381146585246,
        "cc_by_type": {
         "choice": 49.152542372881356,
         "noul": 33.33333333333333,
         "score": -13.193732266458946
        },
        "low_n": true,
        "n": 10,
        "n_by_type": {
         "choice": 5,
         "noul": 3,
         "score": 2
        },
        "score": 23.097381146585246
       }
      },
      "type": {
       "choice": {
        "accuracy": 0.7186666666666667,
        "cc": 63.661761610440315,
        "cc_by_type": {
         "choice": 63.661761610440315
        },
        "low_n": false,
        "n": 750,
        "n_by_type": {
         "choice": 750
        },
        "score": 63.661761610440315
       },
       "noul": {
        "accuracy": 0.7946666666666666,
        "cc": 58.93333333333332,
        "cc_by_type": {
         "noul": 58.93333333333332
        },
        "low_n": false,
        "n": 375,
        "n_by_type": {
         "noul": 375
        },
        "score": 58.93333333333332
       },
       "score": {
        "accuracy": 0.5466666666666666,
        "cc": 36.46304903717563,
        "cc_by_type": {
         "score": 36.46304903717563
        },
        "low_n": false,
        "n": 375,
        "n_by_type": {
         "score": 375
        },
        "score": 36.46304903717563
       }
      },
      "usecase": {
       "advertising": {
        "accuracy": 0.7272727272727273,
        "cc": 60.11455074722698,
        "cc_by_type": {
         "choice": 68,
         "noul": 50,
         "score": 62.343652241680935
        },
        "low_n": true,
        "n": 22,
        "n_by_type": {
         "choice": 8,
         "noul": 8,
         "score": 6
        },
        "score": 60.11455074722698
       },
       "code_linting": {
        "accuracy": 0.7083333333333334,
        "cc": 52.98927870838881,
        "cc_by_type": {
         "choice": 81.36645962732919,
         "noul": 77.77777777777777,
         "score": -0.17640127994054122
        },
        "low_n": true,
        "n": 24,
        "n_by_type": {
         "choice": 7,
         "noul": 9,
         "score": 8
        },
        "score": 52.98927870838881
       },
       "customer_support": {
        "accuracy": 0.7142857142857143,
        "cc": 63.07909063485587,
        "cc_by_type": {
         "choice": 51.21951219512195,
         "noul": 50,
         "score": 88.01775970944567
        },
        "low_n": true,
        "n": 14,
        "n_by_type": {
         "choice": 8,
         "noul": 4,
         "score": 2
        },
        "score": 63.07909063485587
       },
       "demand_forecasting": {
        "accuracy": 0.4117647058823529,
        "cc": 1.3405683540335727,
        "cc_by_type": {
         "choice": 29.82456140350877,
         "noul": -33.333333333333336,
         "score": 7.530476991925283
        },
        "low_n": true,
        "n": 17,
        "n_by_type": {
         "choice": 11,
         "noul": 3,
         "score": 3
        },
        "score": 1.3405683540335727
       },
       "ecommerce_marketplaces": {
        "accuracy": 0.6470588235294118,
        "cc": 22.50908469774433,
        "cc_by_type": {
         "choice": 71.0144927536232,
         "noul": -33.333333333333336,
         "score": 29.846094672943135
        },
        "low_n": true,
        "n": 17,
        "n_by_type": {
         "choice": 9,
         "noul": 3,
         "score": 5
        },
        "score": 22.50908469774433
       },
       "feature_extraction": {
        "accuracy": 0.6842105263157895,
        "cc": -2.182225061345368,
        "cc_by_type": {
         "choice": 49.685534591194966,
         "noul": 100,
         "score": -156.23220977523107
        },
        "low_n": true,
        "n": 19,
        "n_by_type": {
         "choice": 10,
         "noul": 6,
         "score": 3
        },
        "score": 0
       },
       "financial_crime": {
        "accuracy": 0.8,
        "cc": 65.99713161742129,
        "cc_by_type": {
         "choice": 90.19607843137256,
         "noul": 71.42857142857142,
         "score": 36.36674499231988
        },
        "low_n": false,
        "n": 30,
        "n_by_type": {
         "choice": 13,
         "noul": 7,
         "score": 10
        },
        "score": 65.99713161742129
       },
       "gaming": {
        "accuracy": 0.5,
        "cc": 41.51345502838387,
        "cc_by_type": {
         "choice": 37.2822299651568,
         "noul": 50,
         "score": 37.258135119994826
        },
        "low_n": true,
        "n": 22,
        "n_by_type": {
         "choice": 12,
         "noul": 4,
         "score": 6
        },
        "score": 41.51345502838387
       },
       "insurance_claims": {
        "accuracy": 0.6,
        "cc": 55.0828385149075,
        "cc_by_type": {
         "choice": 58.33333333333333,
         "noul": 60.00000000000001,
         "score": 46.91518221138916
        },
        "low_n": true,
        "n": 15,
        "n_by_type": {
         "choice": 6,
         "noul": 5,
         "score": 4
        },
        "score": 55.0828385149075
       },
       "knowledge_graphs": {
        "accuracy": 0.5909090909090909,
        "cc": 39.60594005800316,
        "cc_by_type": {
         "choice": 35.76017130620985,
         "noul": 66.66666666666667,
         "score": 16.390982201132953
        },
        "low_n": true,
        "n": 22,
        "n_by_type": {
         "choice": 10,
         "noul": 6,
         "score": 6
        },
        "score": 39.60594005800316
       },
       "lead_generation": {
        "accuracy": 0.76,
        "cc": 59.68092147823734,
        "cc_by_type": {
         "choice": 100,
         "noul": 55.55555555555556,
         "score": 23.48720887915645
        },
        "low_n": true,
        "n": 25,
        "n_by_type": {
         "choice": 11,
         "noul": 9,
         "score": 5
        },
        "score": 59.68092147823734
       },
       "legal_compliance": {
        "accuracy": 0.5,
        "cc": 26.22694115574735,
        "cc_by_type": {
         "choice": 35.483870967741936,
         "noul": 33.33333333333333,
         "score": 9.863619166166792
        },
        "low_n": true,
        "n": 10,
        "n_by_type": {
         "choice": 4,
         "noul": 3,
         "score": 3
        },
        "score": 26.22694115574735
       },
       "llm_guardrails": {
        "accuracy": 0.7777777777777778,
        "cc": 68.69185757191399,
        "cc_by_type": {
         "choice": 88.48368522072937,
         "noul": 60.00000000000001,
         "score": 57.59188749501257
        },
        "low_n": true,
        "n": 27,
        "n_by_type": {
         "choice": 11,
         "noul": 5,
         "score": 11
        },
        "score": 68.69185757191399
       },
       "model_routing": {
        "accuracy": 0.8202764976958525,
        "cc": 69.08316612831908,
        "cc_by_type": {
         "choice": 74.28571428571429,
         "noul": 93.93939393939394,
         "score": 39.024390159849
        },
        "low_n": false,
        "n": 217,
        "n_by_type": {
         "choice": 101,
         "noul": 66,
         "score": 50
        },
        "score": 69.08316612831908
       },
       "moderation_trust_safety": {
        "accuracy": 0.7222222222222222,
        "cc": 77.16862866609263,
        "cc_by_type": {
         "choice": 65.4510556621881,
         "noul": 100,
         "score": 66.0548303360898
        },
        "low_n": true,
        "n": 18,
        "n_by_type": {
         "choice": 11,
         "noul": 2,
         "score": 5
        },
        "score": 77.16862866609263
       },
       "recruiting": {
        "accuracy": 0.7777777777777778,
        "cc": 84.59956437228902,
        "cc_by_type": {
         "choice": 100,
         "noul": 100,
         "score": 53.79869311686706
        },
        "low_n": true,
        "n": 18,
        "n_by_type": {
         "choice": 8,
         "noul": 3,
         "score": 7
        },
        "score": 84.59956437228902
       },
       "risk_assessment": {
        "accuracy": 0.5555555555555556,
        "cc": 20.430023754833893,
        "cc_by_type": {
         "choice": 55.665024630541865,
         "noul": -50,
         "score": 55.62504663395982
        },
        "low_n": true,
        "n": 18,
        "n_by_type": {
         "choice": 9,
         "noul": 4,
         "score": 5
        },
        "score": 20.430023754833893
       },
       "scientific_discovery": {
        "accuracy": 0.6086956521739131,
        "cc": 42.86130199817126,
        "cc_by_type": {
         "choice": 56.52173913043478,
         "noul": 33.33333333333333,
         "score": 38.72883353074567
        },
        "low_n": true,
        "n": 23,
        "n_by_type": {
         "choice": 9,
         "noul": 6,
         "score": 8
        },
        "score": 42.86130199817126
       },
       "search_retrieval": {
        "accuracy": 0.9166666666666666,
        "cc": 93.80428848041588,
        "cc_by_type": {
         "choice": 89.3238434163701,
         "noul": 100,
         "score": 92.0890220248775
        },
        "low_n": true,
        "n": 24,
        "n_by_type": {
         "choice": 12,
         "noul": 4,
         "score": 8
        },
        "score": 93.80428848041588
       }
      }
     },
     "breakdowns_equated": false,
     "ci95": {
      "B": 1000,
      "C": [
       77.28120639560315,
       84.64714278257424
      ],
      "I": [
       50.218617607373794,
       58.73980484407819
      ],
      "capability": [
       64.3485197527183,
       71.10803595803117
      ],
      "composite": {
       "A": [
        44.392703292458805,
        46.89203249517508
       ],
       "B": [
        43.17507195851998,
        46.534331841823644
       ],
       "C": [
        31.09870948973084,
        44.91059204066695
       ]
      },
      "seed": 26
     },
     "ci95_n": 1200,
     "complete": true,
     "equated": null,
     "equating_flag": null,
     "equating_offset": null,
     "full_set_api": false,
     "g_med_used": 9.839526770355711,
     "lane": "selfhosted",
     "metric_basis": "S u P",
     "n_items": 1500,
     "n_public": 300,
     "n_sealed_side": 1200,
     "on_S": {
      "calibration": 81.82393852673533,
      "capability": 68.39271102371643,
      "intelligence": 54.96148352069754
     },
     "per_type": {
      "choice": {
       "accuracy": 0.7186666666666667,
       "calibration": 81.79703955151288,
       "cc": 63.661761610440315,
       "cc_by_type": {
        "choice": 63.661761610440315
       },
       "cc_tiered": 62.570157201540155,
       "cc_tiered_open": 62.113835115843074,
       "cc_tiered_sealed": 63.02647928723724,
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 63.661761610440315
      },
      "noul": {
       "accuracy": 0.7946666666666666,
       "calibration": 80.88511475193768,
       "cc": 58.93333333333332,
       "cc_by_type": {
        "noul": 58.93333333333332
       },
       "cc_tiered": 63.34309233547301,
       "cc_tiered_open": 71.3768115942029,
       "cc_tiered_sealed": 55.309373076743114,
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 58.93333333333332
      },
      "score": {
       "accuracy": 0.5466666666666666,
       "calibration": 82.78966127675545,
       "cc": 36.46304903717563,
       "cc_by_type": {
        "score": 36.46304903717563
       },
       "cc_tiered": 38.971201025079466,
       "cc_tiered_open": 43.862911895878305,
       "cc_tiered_sealed": 34.079490154280634,
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.46304903717563
      }
     },
     "raw_on_A": {
      "calibration": 81.0046715261283,
      "capability": 66.7025205730405,
      "intelligence": 52.400369619952706
     },
     "set": "S+P"
    },
    "validity": {
     "choice": {
      "invalid_rate": 0,
      "n": 750
     },
     "noul": {
      "invalid_rate": 0,
      "n": 375
     },
     "score": {
      "invalid_rate": 0,
      "n": 375
     }
    },
    "views": {
     "Balanced 33:33:33": 42.095287401061526,
     "Emphasis on Accuracy 60:20:20": 42.2473928103636,
     "Emphasis on Cost 20:20:60": 46.3627340067867,
     "Emphasis on Speed 20:60:20": 38.42054978162839,
     "Intelligence only": 42.477623199386635
    }
   },
   "rank": 22,
   "ranked": true,
   "ranks": {
    "A": 22,
    "B": 55,
    "C": 75,
    "capability": 113
   },
   "remeasure": {
    "disclosure": "Chon family, same author as the 12B sibling. This uses the author's documented --no-think mode; the author's GPU default thinks first on up to 45 % of questions, while that think pass never runs here. GPU re-time in the same --no-think serving configuration as the previous CPU measurement; the default think pass is not exercised. The yes/no commit rule [0.199, 0.801] moves P(yes) just outside the scorer's decisive band: the author's own mechanism. Fifteen states over 6,000 tokens are truncated keeping the start, answered rather than refused. The 231-public-item check was a sanity gate, not an independence check: the author used public items to choose serving settings. ATTEMPT HISTORY: an earlier think-on GPU attempt stopped at the public projection gate before any sealed byte and contributes no scored output. This no-think GPU run is the sole successful replacement measurement. All 1,500 items answered, zero output tokens and zero thinking items. GPU telemetry sampler produced an empty CSV; utilisation and clock samples are unavailable. Scores AND times both come from this GPU run, replacing the CPU measurement in full (scores and times). CPU and GPU probabilities differ on 1,049 of 1,500 items (9 top answers changed), attributed to device numerics; Intelligence and Calibration therefore moved slightly.",
    "draw": "v1.6-regular-20261012",
    "kind": "re-measure on the same items",
    "replaces": {
     "board_release": "v1.7.2",
     "endpoint_kind": "cpu",
     "in_full": "scores and times"
    },
    "supersedes_base_row": {
     "endpoint_kind": "cpu",
     "key": "chon-1.0-e4b",
     "raw_sha256": "e5258cc6e1d4cfa34fd4b2ee2ee1602941e3848b10012e7b3273db704f05d42f"
    }
   },
   "remeasured_in": "v1.7.3",
   "repo": "https://huggingface.co/ninhnv/chon-1.0-e4b",
   "scores": {
    "A": 74.71792231346033,
    "B": 69.87493152358529,
    "C": 63.90410706432046
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2972869430000401,
    "p50_s_raw": 0.07364347150002004,
    "p95_s_adjusted": 0.4526958840999441,
    "p95_s_raw": 0.15134794204997207
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 68.76208788485228,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 22.95293032122352
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 68.76208788485228
      },
      "calibration_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 27.454535856883968,
       "cc_by_type": {
        "choice": 36.37327677624603,
        "noul": 43.99999999999999,
        "score": 1.9903307944058812
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 27.454535856883968
      },
      "composite_gen": {
       "accuracy": 0.6138613861386139,
       "cc": 32.94722453341732,
       "cc_by_type": {
        "choice": 64.22764227642277,
        "noul": 3.703703703703698,
        "score": 30.9103276201255
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 32.94722453341732
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 75.47212152270147,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 66.4163645681044
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 75.47212152270147
      },
      "jaggedness": {
       "accuracy": 0.5444444444444444,
       "cc": 28.18937967019838,
       "cc_by_type": {
        "choice": 49.4706448508181,
        "noul": 35.13513513513513,
        "score": -0.03764097535807753
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 28.18937967019838
      },
      "llm_router": {
       "accuracy": 0.7955271565495208,
       "cc": 68.50572362481046,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 87.62886597938144,
        "score": 52.756668019114386
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.50572362481046
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": 9.353269968593969,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": 0,
        "score": 18.382390550943196
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 9.353269968593969
      },
      "ml_usecase": {
       "accuracy": 0.5185185185185185,
       "cc": 22.467801219021123,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": 25,
        "score": 16.630207780774708
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 22.467801219021123
      },
      "multilingual": {
       "accuracy": 0.76,
       "cc": 64.33087236399763,
       "cc_by_type": {
        "choice": 67.72089990218454,
        "noul": 69.44444444444444,
        "score": 55.82727274536392
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 64.33087236399763
      },
      "negation_pair_gen": {
       "accuracy": 0.5813953488372093,
       "cc": 24.425345430896403,
       "cc_by_type": {
        "choice": 50,
        "noul": 13.33333333333333,
        "score": 9.942702959355888
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 24.425345430896403
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 69.56216308507264,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100,
        "score": 38.71651928524796
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 69.56216308507264
      },
      "usecase": {
       "accuracy": 0.7272727272727273,
       "cc": 63.48416412724288,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 75,
        "score": 42.41327669545411
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 63.48416412724288
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": 25.42541725612485,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 100,
        "score": -28.48565299353021
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 25.42541725612485
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 28.77176040853584,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 40.38594174788095
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 28.77176040853584
      },
      "de": {
       "accuracy": 0.875,
       "cc": 82.52419734145211,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100,
        "score": 79.43099910400234
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 82.52419734145211
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 51.736749506717594,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 88.72980159277847
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 51.736749506717594
      },
      "en": {
       "accuracy": 0.6750651607298002,
       "cc": 50.042168051204726,
       "cc_by_type": {
        "choice": 62.21671690606654,
        "noul": 56.11510791366907,
        "score": 31.794679333878573
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 50.042168051204726
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 68.4123952224834,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 5.237185667450195
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 68.4123952224834
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 75.57043764769237,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 100,
        "score": 92.28508343488038
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 75.57043764769237
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 73.30492499217526,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 83.44418674123168
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 73.30492499217526
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 60.81012825146729,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 50,
        "score": 49.19016129071473
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 60.81012825146729
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8,
       "cc": 71.31966110559327,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 27.988601399398682
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.31966110559327
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 44.623157933573054,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 80,
        "score": 25.354779598971746
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 44.623157933573054
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 77.1063487151395,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 31.319046145418493
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 77.1063487151395
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.22647997234988,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.67943991704968
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.22647997234988
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 46.48059703909075,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 33.33333333333333,
        "score": 32.08248375796491
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.48059703909075
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 35.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 35.483870967741936
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 67.8565202998609,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 51.18699636297672
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 67.8565202998609
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 75.25423065037852,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 68.19050574017503
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 75.25423065037852
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 41.70136243631472,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 60.00000000000001,
        "score": 41.699831989795186
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 41.70136243631472
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 39.991216309885566,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50,
        "score": 13.451909799221905
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 39.991216309885566
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 82.94126933960327,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 81.08187253493887
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 82.94126933960327
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 23.14433039400589,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": -13.052884524197017
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 23.14433039400589
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7146666666666667,
       "cc": 63.145104192579275,
       "cc_by_type": {
        "choice": 63.145104192579275
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 63.145104192579275
      },
      "noul": {
       "accuracy": 0.8,
       "cc": 60.00000000000001,
       "cc_by_type": {
        "noul": 60.00000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 60.00000000000001
      },
      "score": {
       "accuracy": 0.5466666666666666,
       "cc": 36.2339257489653,
       "cc_by_type": {
        "score": 36.2339257489653
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.2339257489653
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 60.101605006991946,
       "cc_by_type": {
        "choice": 68,
        "noul": 50,
        "score": 62.30481502097584
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 60.101605006991946
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 60.348527770088616,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": -0.3208763170633455
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 60.348527770088616
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 57.87233828865274,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50,
        "score": 88.65766527246228
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 57.87233828865274
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 1.4712199418218834,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -33.333333333333336,
        "score": 7.922431755290216
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 1.4712199418218834
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 22.322067113994194,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 29.285041921692713
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.322067113994194
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": -2.025699029138385,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 100,
        "score": -155.76263167861012
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.8,
       "cc": 65.86965014559918,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 71.42857142857142,
        "score": 35.9843005768536
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 65.86965014559918
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 41.6041493747939,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": 50,
        "score": 37.53021815922488
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 41.6041493747939
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 55.36353402803824,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 60.00000000000001,
        "score": 47.757268750781364
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 55.36353402803824
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 39.91024541947309,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 17.303898285542772
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 39.91024541947309
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 67.09757456025866,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 23.514945902998228
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.09757456025866
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 25.88884130225715,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 8.849319605696182
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 25.88884130225715
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 68.52260665797634,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 57.084134753199635
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.52260665797634
      },
      "model_routing": {
       "accuracy": 0.8202764976958525,
       "cc": 68.85491275363961,
       "cc_by_type": {
        "choice": 74.28571428571429,
        "noul": 93.93939393939394,
        "score": 38.3396300358106
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 68.85491275363961
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 72.69549810092742,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 100,
        "score": 64.1517534198648
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 72.69549810092742
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 84.26213346107589,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 52.786400383227694
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 84.26213346107589
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 20.29024281956677,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -50,
        "score": 55.20570382815845
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 20.29024281956677
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 42.767119578393,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 38.4462862714109
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 42.767119578393
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 93.82784396416025,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 92.15968847611069
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 93.82784396416025
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.8437169146738,
      84.20183067549304
     ],
     "I": [
      50.69795588073073,
      59.08516560601005
     ],
     "capability": [
      64.39465991922013,
      71.06078318016968
     ],
     "composite": null,
     "seed": 26
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.517233540399,
     "capability": 68.5028832181483,
     "intelligence": 55.488532895897606
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7146666666666667,
      "calibration": 79.8342624375766,
      "cc": 63.145104192579275,
      "cc_by_type": {
       "choice": 63.145104192579275
      },
      "cc_tiered": 62.22976920966785,
      "cc_tiered_open": 62.113835115843074,
      "cc_tiered_sealed": 62.34570330349263,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 63.145104192579275
     },
     "noul": {
      "accuracy": 0.8,
      "calibration": 81.95621266341794,
      "cc": 60.00000000000001,
      "cc_by_type": {
       "noul": 60.00000000000001
      },
      "cc_tiered": 65.44585913389199,
      "cc_tiered_open": 74.85507246376812,
      "cc_tiered_sealed": 56.03664580401585,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 60.00000000000001
     },
     "score": {
      "accuracy": 0.5466666666666666,
      "calibration": 82.76122552020249,
      "cc": 36.2339257489653,
      "cc_by_type": {
       "score": 36.2339257489653
      },
      "cc_tiered": 38.78997034413298,
      "cc_tiered_open": 43.71332471144403,
      "cc_tiered_sealed": 33.86661597682193,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.2339257489653
     }
    },
    "raw_on_A": {
     "calibration": 80.30010079105372,
     "capability": 66.54892476546081,
     "intelligence": 52.7977487398679
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 72.69672333636456,
    "Emphasis on Accuracy 60:20:20": 64.67400231428026,
    "Emphasis on Cost 20:20:60": 76.67160277594813,
    "Emphasis on Speed 20:60:20": 78.35435507464084,
    "Intelligence only": 55.48853289589761
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "mrmps (@michael_chomsky)",
   "axes": {
    "calibration": 91.33,
    "cost": 58.89,
    "intelligence": 65.16,
    "speed": 78.21
   },
   "calibration": {
    "score": 91.33
   },
   "capability": 78.25,
   "class": "jev-service",
   "composite_ci95": {
    "A": [
     67.79,
     74.01
    ],
    "B": [
     65.25,
     74.13
    ],
    "C": [
     59.49,
     74.01
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (proxy tokens, I-2): operator list price: higher of 19 Sep plan cost and 26 Sep usage tariff USD 0.042/M input (rule 1.2); no exact base-model floor applies",
    "kind": "estimate",
    "usd_per_1000": 0.02345172413793104
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "classifier.dev (fast tier)",
   "endpoint_condition": "the operator's hosted API",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 67.78,
    "I_sealed": 57.39,
    "base": 65.16,
    "per_type_split": {
     "open|choice": {
      "cc": 79.2975,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100,
       "hard": 73.5766,
       "judge": 76.269,
       "standard": 84.9309
      }
     },
     "open|noul": {
      "cc": 57.942,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.5217,
       "judge": 40,
       "standard": 66.6667
      }
     },
     "open|score": {
      "cc": 66.1139,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 72.6364,
       "hard": 64.3333,
       "judge": 75.5254,
       "standard": 52.2963
      }
     },
     "sealed|choice": {
      "cc": 74.0974,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 83.3179,
       "hard": 68.9755,
       "judge": 68.3417,
       "standard": 88.3646
      }
     },
     "sealed|noul": {
      "cc": 47.1925,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 36.3636,
       "judge": 50,
       "standard": 50
      }
     },
     "sealed|score": {
      "cc": 50.894,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 79.6944,
       "hard": 31.0482,
       "judge": 48.7273,
       "standard": 79.4356
      }
     }
    }
   },
   "jevbench_score": 71.35,
   "key": "classifier-dev-fast",
   "last_measured_on": "2026-10-06",
   "licence": "MIT (code); hosted service",
   "listing": "wrapper",
   "method": "v1.7",
   "not_ranked_because": "Wrapper that serves Jev: listed, never ranked.",
   "open": "no",
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.33,
     "cost": 58.89,
     "intelligence": 65.16,
     "speed": 79.54
    },
    "board_release": "v1.6.43",
    "capability": 78.25,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      68.03,
      74.31
     ],
     "B": [
      65.43,
      74.36
     ],
     "C": [
      59.7,
      74.31
     ]
    },
    "jevbench_score": 71.62,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": 71.62,
     "B": 70.23,
     "C": 71.62
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.806,
     "p50_s_raw": 0.806,
     "p95_s_adjusted": 1.381,
     "p95_s_raw": 1.381
    },
    "views": null
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://classifier.dev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.35,
    "B": 70.02,
    "C": 71.35
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.98,
    "p50_s_raw": 0.98,
    "p95_s_adjusted": 1.542,
    "p95_s_raw": 1.542
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "cc0da2fbc545cd44056fce4dbc8b1e17b6adbbe73146aa905c9abcfa6b0d3a87"
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)",
   "axes": {
    "calibration": 22.615652963639242,
    "cost": 85.79078096700756,
    "intelligence": 7.802105373202749,
    "speed": 93.07921198047904
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.5367426710097716,
      "ece_score": 0,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 0,
      "score_before_time_limit": 0,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.1671159029649596,
      "brier": 0.3670596395956874,
      "ece": 0.34489191374663086,
      "n": 371,
      "score": 31.021617250673827,
      "score_before_time_limit": 31.021617250673827,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.5121951219512195,
      "interval90_width": 0.28523035230352306,
      "n": 369,
      "nrps": 0.263493167195122,
      "score": 36.825341640243906,
      "score_before_time_limit": 36.825341640243906,
      "time_limit_failed_share": 0,
      "top_ece": 0.5007441734417342
     }
    },
    "score": 22.615652963639242
   },
   "capability": 15.208879168420996,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 727,075 input + 0 output tokens over 1,477 decisions = 492.264725796 tokens/decision. GPU-time cost USD 0.00297605281646624/1,000; old list-price cost USD 0.00984529451591063/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00297605281646624/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.009694333333333333
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.002976052816466239
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ClassOne Gemma 4 E2B (Gemma-4-E2B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,477 of 1,500 answered. Weights devops-thiago/classone-gemma4-e2b (sha cd502aa33d288e67f16937c2e125f2297ab7b4ad), Apache-2.0, public and ungated; code github.com/devops-thiago/class-one, Apache-2.0. ONE DOCUMENTED MAPPING OF OURS, AND IT IS THE ONLY ONE. The author's server keys SCORE probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1'. The UNAMENDED smoke gate REFUSED the first attempt at check score-keys-are-indices, before any sealed item was sent, and the server was torn down with the card verified free. The 1-based convention - which their own schemas.py documents only as 'Probability distribution across each ordered rubric level', without stating the convention - was then established on the AUTHOR'S OWN SERVER with SYNTHETIC requests across every rubric size their own validator allows (2..10): exactly the keys '1'..'n' in order every time, never '0' and never a key above n. It is corroborated independently by their own `score` field being a 1-based probability-weighted mean of the same distribution (1.0185 on a 2-level rubric, 1.6147 on a 10-level one, 3.1155 where the mass sits on key 3; a 0-based convention would have given 0.0185), and a reversed rubric moved the mass to a different key, so 'k' means the k-th level of the criteria array AS SENT - which is what JevBench's 'k-1' means. classone_reindex_r58.ClassOneReindexAdapter therefore relabels score keys k -> k-1 and NOTHING else: the request body, the answer mapping, the noul range check and the choice-must-be-one-of-the-item's-labels rule are the UNCHANGED vendored typesafe adapter's, reused by calling its own run(). An item whose score key set is not exactly {'1'..'n'} is REFUSED and recorded invalid rather than rescored on a guess; none was. choice and noul are untouched. The gate then ran AMENDED in exactly that one check, requiring the key set to be exactly {'1'..'n'} - stricter than the original, because it also pins the count - with the other seven byte-identical, and passed. PREREG-R58-AMENDMENT-1.md (sha256 in PREREG-R58-AMENDMENT-1.md.sha256), receipt PROBE-classone-e2b.json. No file of the author's was edited and no prompt of ours exists. THE SUBMIT FORM CARRIED NO DESCRIPTION FOR EITHER CLASSONE ROW, so everything about how it is run comes from the author's own repository: install line README Quickstart step 1 (`pip install -e \".[dev]\"`, in an isolated venv - torch 2.14.1+cu130, transformers 5.19.0, fastapi 0.142.2), serve line step 3 (`uvicorn classone.server.app:app`), and the checkpoint selected the way their own src/classone/server/app.py reads it (CLASSONE_BASE_MODEL). THEIR SERVER REGISTERS POST /v1/systemone ITSELF (app.py:115, next to /v1/decide and /v1/classone) and their DecisionRequest/DecisionResponse in src/classone/schemas.py are the TypeSafe System One shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. output_tokens is 0 by their own schema comment ('In System 1 models this is 0 because there is no autoregressive decode'). THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW REFUSES THEM at a limit far below them: all 23 came back as 'HTTP 400: Prompt length (81338 tokens) exceeds maximum limit (8192)', an 8,192-token ceiling the author's code enforces with an explicit error rather than truncating. The 23 invalid are the system's own refusal, counted wrong, and no other item failed. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 58, benchmarkheaven.com/submit ref C4D7AACD (bh-submit 4567a065-e912-40df-9bc1-68fb37cc7bba)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB",
   "intelligence": {
    "I_open": 8.900914683249555,
    "I_sealed": 6.703296063155943,
    "base": 7.802105373202749,
    "excess": -0.3757690223446315,
    "gap": 2.1976186200936123,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 26.702744049748667,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 18.270270270270274,
       "hard": 16.955017301038072,
       "judge": 28.80685563612393,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -13.043478260869568,
       "judge": -40,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.08424242424244,
       "hard": -7.722870370370338,
       "judge": -13.495593220338996,
       "standard": -43.26740740740745
      }
     },
     "sealed|choice": {
      "cc": 20.109888189467828,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 30.82551178774667,
       "hard": 12.952488687782791,
       "judge": 20.856610800744896,
       "standard": 27.946791476782867
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 15.999999999999993,
       "hard": -37.86407766990292,
       "judge": 2.6315789473684292,
       "standard": 23.943661971830977
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 14.937446808510646,
       "hard": -32.15683823529416,
       "judge": -35.42389221556888,
       "standard": -33.18258883248737
      }
     }
    }
   },
   "jevbench_score": 0.500014420840021,
   "key": "classone-gemma4-e2b",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "devops-thiago/classone-gemma4-e2b@cd502aa33d288e67f16937c2e125f2297ab7b4ad",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5889967637540453,
    "decisive_rate": 0.8328840970350404,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 22.615652963639242,
     "cost": 70.20313863040258,
     "intelligence": 7.802105373202749,
     "speed": 93.2878888255429
    },
    "board_release": "v1.6.43",
    "capability": 15.208879168420996,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 137,
    "capability_rank": 167,
    "composite_ci95": {
     "A": [
      0.24070026102449685,
      1.3152079969468897
     ],
     "B": [
      0.17590250514236588,
      1.0549899064852435
     ],
     "C": [
      0.16715295904478944,
      0.9133388867686731
     ]
    },
    "jevbench_score": 0.49351800738768026,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 163,
    "rank": 206,
    "ranks": {
     "A": 206,
     "B": 204,
     "C": 206,
     "capability": 239
    },
    "scores": {
     "A": 0.49351800738768026,
     "B": 0.37400131309297324,
     "C": 0.34272083846366685
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20868116198689676,
     "p50_s_raw": 0.029340580993448384,
     "p95_s_adjusted": 0.2247645789931994,
     "p95_s_raw": 0.0373822894965997
    },
    "views": {
     "Balanced 33:33:33": 0.4770150324027566,
     "Emphasis on Accuracy 60:20:20": 0.2973210912399719,
     "Emphasis on Cost 20:20:60": 0.6703202939761591,
     "Emphasis on Speed 20:60:20": 0.6973896308140026,
     "Intelligence only": 0.1899745505796208
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 248,
   "ranked": true,
   "ranks": {
    "A": 248,
    "B": 246,
    "C": 248,
    "capability": 280
   },
   "repo": "https://huggingface.co/devops-thiago/classone-gemma4-e2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.500014420840021,
    "B": 0.37697064517172496,
    "C": 0.347232236694459
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20895755998790264,
    "p50_s_raw": 0.02947877999395132,
    "p95_s_adjusted": 0.23551616103213746,
    "p95_s_raw": 0.04275808051606873
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-E2B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      19.456713582756297,
      25.97574684722229
     ],
     "I": [
      5.95996205912109,
      11.37620576911391
     ],
     "capability": [
      13.386066785667875,
      17.91536010233214
     ],
     "composite": null,
     "seed": 26
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 22.615652963639242,
     "capability": 15.208879168420996,
     "intelligence": 7.802105373202749
    },
    "per_type": {
     "choice": {
      "accuracy": 0.38266666666666665,
      "calibration": 0,
      "cc": 21.051883709254156,
      "cc_by_type": {
       "choice": 21.051883709254156
      },
      "cc_tiered": 23.40631611960825,
      "cc_tiered_open": 26.702744049748667,
      "cc_tiered_sealed": 20.109888189467828,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 21.051883709254156
     },
     "noul": {
      "accuracy": 0.48533333333333334,
      "calibration": 31.021617250673827,
      "cc": -2.9333333333333322,
      "cc_by_type": {
       "noul": -2.9333333333333322
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2853333333333333,
      "calibration": 36.825341640243906,
      "cc": -21.676596491227972,
      "cc_by_type": {
       "score": -21.676596491227972
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 26.647352805544053,
     "capability": 18.967337110684245,
     "intelligence": 11.287321415824435
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.48513818067516945,
    "Emphasis on Accuracy 60:20:20": 0.29919460803641246,
    "Emphasis on Cost 20:20:60": 0.7001592621970353,
    "Emphasis on Speed 20:60:20": 0.7075876434264765,
    "Intelligence only": 0.1899745505796208
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)",
   "axes": {
    "calibration": 48.5204641422835,
    "cost": 82.05537563260856,
    "intelligence": 28.182424563225457,
    "speed": 91.65069544774573
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.27998611987381716,
      "ece_score": 44.00277602523657,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 44.00277602523657,
      "score_before_time_limit": 44.00277602523657,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.06593406593406592,
      "brier": 0.25247848434065967,
      "ece": 0.2512450549450551,
      "n": 364,
      "score": 49.750989010988974,
      "score_before_time_limit": 49.750989010988974,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.7146739130434783,
      "interval90_width": 0.3179347826086956,
      "n": 368,
      "nrps": 0.19245016957880431,
      "score": 51.80762739062497,
      "score_before_time_limit": 51.80762739062497,
      "time_limit_failed_share": 0,
      "top_ece": 0.38569864130434806
     }
    },
    "score": 48.5204641422835
   },
   "capability": 38.35144435275448,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 4B active parameters; throughput 80585.75 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 792,614 input + 0 output tokens over 1,500 decisions = 528.409333333 tokens/decision. GPU-time cost USD 0.00396418711495447/1,000; old list-price cost USD 0.0105681866666667/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00396418711495447/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003964187114954465
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ClassOne Gemma 4 E4B (Gemma-4-E4B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "The author's own README Quickstart install line (pip install -e '.[dev]') at code pin github.com/devops-thiago/class-one@6cca132b and their own serve line (uvicorn classone.server.app:app); their server registers POST /v1/systemone itself, so the request and answer shapes are theirs. CLASSONE_BASE_MODEL was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, which their own code accepts, so the revision cannot drift to main and classone_heads.pt resolves locally with no request-time hub fetch. Their hard-coded MAX_TOKENS_PER_REQUEST = 8192 refuses longer prompts with HTTP 400; those items are recorded as invalid, the same way for every system. ONE documented mapping of ours and the only one: their server keys score probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1', so run 58's byte-identical ClassOneReindexAdapter relabels score keys k -> k-1 by calling the unchanged vendored typesafe adapter's own run(), and an item whose key set is not exactly {'1'..'n'} is REFUSED rather than rescored on a guess; choice and noul are untouched. The two already-published ClassOne rows carry exactly this treatment. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 33.31401280028791,
    "I_sealed": 23.050836326163,
    "base": 28.182424563225457,
    "excess": 0.1899318927851983,
    "gap": 10.263176474124911,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 35.64521709768209,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 20.729789241899976,
       "judge": 36.71720500988794,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 41.01932367149759,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 47.82608695652173,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 23.277497631684074,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 38.74636363636365,
       "hard": 25.742037037037047,
       "judge": 16.479661016949144,
       "standard": 20.81074074074073
      }
     },
     "sealed|choice": {
      "cc": 37.362851267778666,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 38.054020231515274,
       "hard": 33.71441134439924,
       "judge": 36.076302572167826,
       "standard": 46.243969676085456
      }
     },
     "sealed|noul": {
      "cc": 31.78965771071034,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 48.71794871794872,
       "hard": 10.606060606060597,
       "judge": 41.66666666666667,
       "standard": 50.877192982456144
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 14.18728971962614,
       "hard": -10.75497222222217,
       "judge": 1.1792035398229705,
       "standard": 4.336081081081044
      }
     }
    }
   },
   "jevbench_score": 16.047288987128645,
   "key": "classone-gemma4-e4b",
   "last_measured_on": "2026-10-10",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "devops-thiago/classone-gemma4-e4b@74d6e08152878a4956ce9ffc323f6c06f13ca9af",
   "noul_decisive": {
    "acc_among_decisive": 0.7382352941176471,
    "decisive": 340,
    "decisive_rate": 0.9340659340659341,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 48.5204641422835,
     "cost": 69.27998573050196,
     "intelligence": 28.182424563225457,
     "speed": 91.7664962211246
    },
    "board_release": "v1.6.43",
    "capability": 38.35144435275448,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 115,
    "capability_rank": 143,
    "composite_ci95": {
     "A": [
      8.912270303019026,
      24.387188108020116
     ],
     "B": [
      7.469929402481821,
      21.841243683018
     ],
     "C": [
      6.189076599318768,
      16.935547297236194
     ]
    },
    "jevbench_score": 15.607101626553092,
    "measured_in": "v1.6.9",
    "method": "v1.6",
    "open_board_rank": 120,
    "rank": 158,
    "ranks": {
     "A": 158,
     "B": 157,
     "C": 159,
     "capability": 207
    },
    "scores": {
     "A": 15.607101626553092,
     "B": 13.587651382976851,
     "C": 10.838265018439648
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.22830128003843128,
     "p50_s_raw": 0.03915064001921564,
     "p95_s_adjusted": 0.29163660332560537,
     "p95_s_raw": 0.07081830166280269
    },
    "views": {
     "Balanced 33:33:33": 15.672226976099346,
     "Emphasis on Accuracy 60:20:20": 12.054100187818078,
     "Emphasis on Cost 20:20:60": 17.712389550075887,
     "Emphasis on Speed 20:60:20": 19.22912598635703,
     "Intelligence only": 8.95354562245959
    }
   },
   "rank": 202,
   "ranked": true,
   "ranks": {
    "A": 202,
    "B": 200,
    "C": 202,
    "capability": 249
   },
   "repo": "https://huggingface.co/devops-thiago/classone-gemma4-e4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.047288987128645,
    "B": 13.852300173054447,
    "C": 11.143950685506002
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2288790670922026,
    "p50_s_raw": 0.0394395335461013,
    "p95_s_adjusted": 0.29876132757635787,
    "p95_s_raw": 0.07438066378817894
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6551724137931034,
       "cc": 48.403611111111104,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 66.66666666666667,
        "score": 11.877499999999985
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 48.403611111111104
      },
      "calibration_gen": {
       "accuracy": 0.42857142857142855,
       "cc": 2.0163035515209473,
       "cc_by_type": {
        "choice": 32.453416149068325,
        "noul": 23.076923076923084,
        "score": -49.481428571428566
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 2.0163035515209473
      },
      "composite_gen": {
       "accuracy": 0.5102040816326531,
       "cc": 24.397270723104047,
       "cc_by_type": {
        "choice": 40.74074074074074,
        "noul": 21.42857142857142,
        "score": 11.02249999999999
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 24.397270723104047
      },
      "fairness": {
       "accuracy": 0.7777777777777778,
       "cc": 78.00062745098039,
       "cc_by_type": {
        "choice": 64.70588235294117,
        "noul": 100,
        "score": 69.296
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 78.00062745098039
      },
      "jaggedness": {
       "accuracy": 0.3837837837837838,
       "cc": 11.209656087483806,
       "cc_by_type": {
        "choice": 18.017600741083836,
        "noul": 23.076923076923084,
        "score": -7.465555555555503
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 11.209656087483806
      },
      "llm_router": {
       "accuracy": 0.6363636363636364,
       "cc": 55.196760619175414,
       "cc_by_type": {
        "choice": 47.05062703204832,
        "noul": 61.64383561643836,
        "score": 56.895819209039544
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 55.196760619175414
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5161290322580645,
       "cc": 23.0149386845039,
       "cc_by_type": {
        "choice": 16.923076923076923,
        "noul": 50,
        "score": 2.1217391304347744
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 23.0149386845039
      },
      "multilingual": {
       "accuracy": 0.5365079365079365,
       "cc": 29.473782295572374,
       "cc_by_type": {
        "choice": 43.573667711598766,
        "noul": 40.22988505747127,
        "score": 4.617794117647089
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 29.473782295572374
      },
      "negation_pair_gen": {
       "accuracy": 0.48214285714285715,
       "cc": -16.955793650793655,
       "cc_by_type": {
        "choice": 19.047619047619047,
        "noul": 30.000000000000004,
        "score": -99.91500000000002
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.38596491228070173,
       "cc": 22.30854605993341,
       "cc_by_type": {
        "choice": 22.30854605993341
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 22.30854605993341
      },
      "perturb_pair": {
       "accuracy": 0.75,
       "cc": 67.99352945343772,
       "cc_by_type": {
        "choice": 64.61336828309307,
        "noul": 71.42857142857142,
        "score": 67.93864864864865
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 67.99352945343772
      },
      "usecase": {
       "accuracy": 0.46835443037974683,
       "cc": 14.865840833856993,
       "cc_by_type": {
        "choice": 42.18539217575643,
        "noul": 12.280701754385959,
        "score": -9.868571428571403
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 14.865840833856993
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6363636363636364,
       "cc": 26.925555555555565,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": 50,
        "score": -15.88999999999996
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 26.925555555555565
      },
      "cs": {
       "accuracy": 0.5,
       "cc": 7.90960451977401,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 7.90960451977401
      },
      "da": {
       "accuracy": 0.4166666666666667,
       "cc": 24.536203155818544,
       "cc_by_type": {
        "choice": 17.159763313609464,
        "noul": 60.00000000000001,
        "score": -3.5511538461538317
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 24.536203155818544
      },
      "de": {
       "accuracy": 0.5333333333333333,
       "cc": 21.35741575994569,
       "cc_by_type": {
        "choice": 56.54797827398914,
        "noul": 11.111111111111116,
        "score": -3.5868421052631883
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 21.35741575994569
      },
      "el": {
       "accuracy": 0.4444444444444444,
       "cc": 0.07861445783131475,
       "cc_by_type": {
        "choice": 27.710843373493972,
        "noul": -19.999999999999996,
        "score": -7.475000000000032
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0.07861445783131475
      },
      "en": {
       "accuracy": 0.49508489722966936,
       "cc": 22.359010078249664,
       "cc_by_type": {
        "choice": 35.3897924845765,
        "noul": 28.57142857142858,
        "score": 3.1158091787439113
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 22.359010078249664
      },
      "es": {
       "accuracy": 0.7857142857142857,
       "cc": 58.303939393939395,
       "cc_by_type": {
        "choice": 73.81818181818181,
        "noul": 100,
        "score": 1.0936363636363633
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 58.303939393939395
      },
      "fi": {
       "accuracy": 0.4444444444444444,
       "cc": 47.183098591549296,
       "cc_by_type": {
        "choice": -5.63380281690141,
        "noul": 100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 47.183098591549296
      },
      "fr": {
       "accuracy": 0.6,
       "cc": 30.50427698300759,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 0,
        "score": 42.28857142857142
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 30.50427698300759
      },
      "hi": {
       "accuracy": 0.5,
       "cc": 33.68955033643892,
       "cc_by_type": {
        "choice": 25.465838509316765,
        "noul": 60.00000000000001,
        "score": 15.602812499999995
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 33.68955033643892
      },
      "id": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 17.741935483870968
      },
      "it": {
       "accuracy": 0.7058823529411765,
       "cc": 57.02479166666668,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 50,
        "score": 21.07437500000001
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 57.02479166666668
      },
      "ja": {
       "accuracy": 0.375,
       "cc": -0.5040061217576456,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": -42.85714285714286,
        "score": 22.645937500000002
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.7272727272727273,
       "cc": 77.51933333333334,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 32.55800000000002
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 77.51933333333334
      },
      "mixed": {
       "accuracy": 1,
       "cc": 98.75666666666667,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.27000000000004
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 98.75666666666667
      },
      "nl": {
       "accuracy": 0.38461538461538464,
       "cc": -15.442971216341691,
       "cc_by_type": {
        "choice": 24.79108635097493,
        "noul": 0,
        "score": -71.12
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.42857142857142855,
       "cc": 34.42622950819672,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 34.42622950819672
      },
      "pl": {
       "accuracy": 0.625,
       "cc": 50.14468360745798,
       "cc_by_type": {
        "choice": 76.38828159160472,
        "noul": 80,
        "score": -5.954230769230784
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 50.14468360745798
      },
      "pt": {
       "accuracy": 0.6111111111111112,
       "cc": 49.75193662758235,
       "cc_by_type": {
        "choice": 45.7286432160804,
        "noul": 66.66666666666667,
        "score": 36.86049999999999
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 49.75193662758235
      },
      "sv": {
       "accuracy": 0.5,
       "cc": -7.282962962962962,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 50,
        "score": -82.96
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.7272727272727273,
       "cc": 23.552514619883052,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 100,
        "score": -103.02666666666664
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 23.552514619883052
      },
      "uk": {
       "accuracy": 0.2727272727272727,
       "cc": -34.453968784838345,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": -16.405384615384612
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.625,
       "cc": 36.97171171171172,
       "cc_by_type": {
        "choice": 35.13513513513514,
        "noul": 100,
        "score": -24.219999999999974
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 36.97171171171172
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5173333333333333,
       "cc": 37.46761098635343,
       "cc_by_type": {
        "choice": 37.46761098635343
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 37.46761098635343
      },
      "noul": {
       "accuracy": 0.6693333333333333,
       "cc": 33.86666666666667,
       "cc_by_type": {
        "noul": 33.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 33.86666666666667
      },
      "score": {
       "accuracy": 0.3413333333333333,
       "cc": 3.9127145359018956,
       "cc_by_type": {
        "score": 3.9127145359018956
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 3.9127145359018956
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": -21.09269082921473,
       "cc_by_type": {
        "choice": 40.691927512355846,
        "noul": 19.999999999999996,
        "score": -123.97000000000004
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": 6.366578355155487,
       "cc_by_type": {
        "choice": -8.01963993453355,
        "noul": 60.00000000000001,
        "score": -32.880624999999995
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 6.366578355155487
      },
      "customer_support": {
       "accuracy": 0.6,
       "cc": 34.63327543424317,
       "cc_by_type": {
        "choice": 51.61290322580645,
        "noul": 25,
        "score": 27.286923076923063
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 34.63327543424317
      },
      "demand_forecasting": {
       "accuracy": 0.23076923076923078,
       "cc": -1.8498271875380354,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": 0,
        "score": -9.163939393939401
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2777777777777778,
       "cc": -8.013566650740566,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -60,
        "score": 10.493461538461535
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.5172413793103449,
       "cc": 16.519821289821277,
       "cc_by_type": {
        "choice": 48.71794871794871,
        "noul": 19.999999999999996,
        "score": -19.158484848484882
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 16.519821289821277
      },
      "financial_crime": {
       "accuracy": 0.5384615384615384,
       "cc": 24.41154782608697,
       "cc_by_type": {
        "choice": 73.91304347826087,
        "noul": 25,
        "score": -25.678399999999968
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 24.41154782608697
      },
      "gaming": {
       "accuracy": 0.3793103448275862,
       "cc": 6.763275341819888,
       "cc_by_type": {
        "choice": 16.831683168316832,
        "noul": 42.85714285714286,
        "score": -39.39900000000003
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 6.763275341819888
      },
      "insurance_claims": {
       "accuracy": 0.2857142857142857,
       "cc": -46.41502645502644,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": -60,
        "score": -99.87999999999995
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.21739130434782608,
       "cc": -6.9660931689176,
       "cc_by_type": {
        "choice": -9.923664122137408,
        "noul": 19.999999999999996,
        "score": -30.97461538461539
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 7.34117073170732,
       "cc_by_type": {
        "choice": 51.21951219512196,
        "noul": 0,
        "score": -29.195999999999998
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 7.34117073170732
      },
      "legal_compliance": {
       "accuracy": 0.5384615384615384,
       "cc": 5.378939247830284,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": 81.71058823529413
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 5.378939247830284
      },
      "llm_guardrails": {
       "accuracy": 0.6551724137931034,
       "cc": 45.51440602211244,
       "cc_by_type": {
        "choice": 72.4770642201835,
        "noul": 50,
        "score": 14.066153846153838
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 45.51440602211244
      },
      "model_routing": {
       "accuracy": 0.7466666666666667,
       "cc": 66.53999293560527,
       "cc_by_type": {
        "choice": 67.83704100777273,
        "noul": 77.27272727272727,
        "score": 54.510210526315795
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 66.53999293560527
      },
      "moderation_trust_safety": {
       "accuracy": 0.5625,
       "cc": 31.290701754385953,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 60.00000000000001,
        "score": -2.970000000000028
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 31.290701754385953
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 37.53701370098354,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 33.33333333333333,
        "score": 9.42846153846154
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 37.53701370098354
      },
      "risk_assessment": {
       "accuracy": 0.6153846153846154,
       "cc": 37.185966981132076,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 100,
        "score": -50.70625
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 37.185966981132076
      },
      "scientific_discovery": {
       "accuracy": 0.28125,
       "cc": 2.9136300828432273,
       "cc_by_type": {
        "choice": -3.585657370517929,
        "noul": 14.28571428571428,
        "score": -1.9591666666666674
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 2.9136300828432273
      },
      "search_retrieval": {
       "accuracy": 0.6818181818181818,
       "cc": 46.84529194088859,
       "cc_by_type": {
        "choice": 57.04057279236276,
        "noul": 66.66666666666667,
        "score": 16.828636363636363
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 46.84529194088859
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      42.95152007629359,
      53.0806033344027
     ],
     "I": [
      22.568457707555886,
      33.716071635910566
     ],
     "capability": [
      33.46902235931629,
      42.57940480198994
     ],
     "composite": null,
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 48.5204641422835,
     "capability": 38.35144435275448,
     "intelligence": 28.182424563225457
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5173333333333333,
      "calibration": 44.00277602523657,
      "cc": 37.46761098635343,
      "cc_by_type": {
       "choice": 37.46761098635343
      },
      "cc_tiered": 36.50403418273038,
      "cc_tiered_open": 35.64521709768209,
      "cc_tiered_sealed": 37.362851267778666,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 37.46761098635343
     },
     "noul": {
      "accuracy": 0.6693333333333333,
      "calibration": 49.750989010988974,
      "cc": 33.86666666666667,
      "cc_by_type": {
       "noul": 33.86666666666667
      },
      "cc_tiered": 36.404490691103966,
      "cc_tiered_open": 41.01932367149759,
      "cc_tiered_sealed": 31.78965771071034,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 33.86666666666667
     },
     "score": {
      "accuracy": 0.3413333333333333,
      "calibration": 51.80762739062497,
      "cc": 3.9127145359018956,
      "cc_by_type": {
       "score": 3.9127145359018956
      },
      "cc_tiered": 11.638748815842037,
      "cc_tiered_open": 23.277497631684074,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 3.9127145359018956
     }
    },
    "raw_on_A": {
     "calibration": 43.42824044580669,
     "capability": 35.61702293170774,
     "intelligence": 27.805805417608788
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.26976331330694,
    "Emphasis on Accuracy 60:20:20": 12.261924843611357,
    "Emphasis on Cost 20:20:60": 19.148965214101477,
    "Emphasis on Speed 20:60:20": 19.75670803528016,
    "Intelligence only": 8.95354562245959
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)",
   "axes": {
    "calibration": 32.62324971930184,
    "cost": 85.83730649437784,
    "intelligence": 5.814527660927306,
    "speed": 92.94182096467813
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.42988044164037864,
      "ece_score": 14.023911671924271,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 14.023911671924271,
      "score_before_time_limit": 14.023911671924271,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.12637362637362637,
      "brier": 0.353035391263736,
      "ece": 0.33436318681318694,
      "n": 364,
      "score": 33.12736263736261,
      "score_before_time_limit": 33.12736263736261,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.7309782608695652,
      "interval90_width": 0.45244565217391297,
      "n": 368,
      "nrps": 0.215767459549366,
      "score": 50.718474848618655,
      "score_before_time_limit": 50.718474848618655,
      "time_limit_failed_share": 0,
      "top_ece": 0.3849315217391305
     }
    },
    "score": 32.62324971930184
   },
   "capability": 19.21888869011457,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 735,765 input + 0 output tokens over 1,500 decisions = 490.51 tokens/decision. GPU-time cost USD 0.00296544438492064/1,000; old list-price cost USD 0.0098102/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00296544438492064/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0029654443849206355
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ClassOne Qwen 3.5 2B (Qwen3.5-2B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "The author's own README Quickstart install line (pip install -e '.[dev]') at code pin github.com/devops-thiago/class-one@6cca132b and their own serve line (uvicorn classone.server.app:app); their server registers POST /v1/systemone itself, so the request and answer shapes are theirs. CLASSONE_BASE_MODEL was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, which their own code accepts, so the revision cannot drift to main and classone_heads.pt resolves locally with no request-time hub fetch. Their hard-coded MAX_TOKENS_PER_REQUEST = 8192 refuses longer prompts with HTTP 400; those items are recorded as invalid, the same way for every system. ONE documented mapping of ours and the only one: their server keys score probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1', so run 58's byte-identical ClassOneReindexAdapter relabels score keys k -> k-1 by calling the unchanged vendored typesafe adapter's own run(), and an item whose key set is not exactly {'1'..'n'} is REFUSED rather than rescored on a guess; choice and noul are untouched. The two already-published ClassOne rows carry exactly this treatment. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 6.892803202604126,
    "I_sealed": 4.736252119250485,
    "base": 5.814527660927306,
    "excess": -7.916693497986072,
    "gap": 2.1565510833536408,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 18.56541234955058,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.297297297297305,
       "hard": 16.955017301038072,
       "judge": 5.075807514831907,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 1.9710144927536213,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": -21.739130434782606,
       "judge": -19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 0.14198276550817734,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.27727272727272,
       "hard": -2.0812037037037046,
       "judge": 4.7081355932203195,
       "standard": -27.328518518518543
      }
     },
     "sealed|choice": {
      "cc": 10.956701395170182,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 19.282511210762333,
       "hard": 9.970319885676586,
       "judge": 3.2012581807112888,
       "standard": 20.399724328049636
      }
     },
     "sealed|noul": {
      "cc": 3.2520549625812754,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 38.46153846153846,
       "hard": -12.121212121212121,
       "judge": 8.333333333333325,
       "standard": 8.771929824561408
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 16.134112149532708,
       "hard": -13.1237222222222,
       "judge": -24.52305309734517,
       "standard": -9.89180180180178
      }
     }
    }
   },
   "jevbench_score": 0.24036918429679566,
   "key": "classone-qwen3.5-2b",
   "last_measured_on": "2026-10-10",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "devops-thiago/classone-qwen3.5-2b@fc15401b6ae6d86b8fdf664b40bfdd8055edf374",
   "noul_decisive": {
    "acc_among_decisive": 0.6226415094339622,
    "decisive": 318,
    "decisive_rate": 0.8736263736263736,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 32.62324971930184,
     "cost": 70.24966415777286,
     "intelligence": 5.814527660927306,
     "speed": 93.11279379161611
    },
    "board_release": "v1.6.43",
    "capability": 19.21888869011457,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 134,
    "capability_rank": 164,
    "composite_ci95": {
     "A": [
      0.07421330468385162,
      1.5542613561325538
     ],
     "B": [
      0.051005668190310965,
      1.2102758221999665
     ],
     "C": [
      0.051537017141563626,
      1.0793481639809404
     ]
    },
    "jevbench_score": 0.23766015025378578,
    "measured_in": "v1.6.9",
    "method": "v1.6",
    "open_board_rank": 168,
    "rank": 212,
    "ranks": {
     "A": 212,
     "B": 212,
     "C": 212,
     "capability": 236
    },
    "scores": {
     "A": 0.23766015025378578,
     "B": 0.1692155731407913,
     "C": 0.16504177100957346
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21041937693953514,
     "p50_s_raw": 0.03020968846976757,
     "p95_s_adjusted": 0.23207849023165178,
     "p95_s_raw": 0.0410392451158259
    },
    "views": {
     "Balanced 33:33:33": 0.2059858256814215,
     "Emphasis on Accuracy 60:20:20": 0.1250037108819809,
     "Emphasis on Cost 20:20:60": 0.29995214102056916,
     "Emphasis on Speed 20:60:20": 0.3095514049527884,
     "Intelligence only": 0.0786327227711625
    }
   },
   "rank": 253,
   "ranked": true,
   "ranks": {
    "A": 253,
    "B": 253,
    "C": 253,
    "capability": 277
   },
   "repo": "https://huggingface.co/devops-thiago/classone-qwen3.5-2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.24036918429679566,
    "B": 0.17030889226810506,
    "C": 0.16692304465055255
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2125162780052051,
    "p50_s_raw": 0.03125813900260255,
    "p95_s_adjusted": 0.23901530014118177,
    "p95_s_raw": 0.0445076500705909
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.4827586206896552,
       "cc": 26.97833333333334,
       "cc_by_type": {
        "choice": 8.333333333333334,
        "noul": 66.66666666666667,
        "score": 5.935000000000013
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 26.97833333333334
      },
      "calibration_gen": {
       "accuracy": 0.3392857142857143,
       "cc": -13.726849020544668,
       "cc_by_type": {
        "choice": 16.14906832298137,
        "noul": -15.384615384615385,
        "score": -41.944999999999986
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.32653061224489793,
       "cc": 0.7510267857142816,
       "cc_by_type": {
        "choice": 0,
        "noul": -7.14285714285714,
        "score": 9.395937499999985
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0.7510267857142816
      },
      "fairness": {
       "accuracy": 0.4444444444444444,
       "cc": -7.00355555555556,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": -21.010666666666687
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.3837837837837838,
       "cc": 5.861599107426817,
       "cc_by_type": {
        "choice": 18.017600741083836,
        "noul": 7.692307692307687,
        "score": -8.125111111111071
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 5.861599107426817
      },
      "llm_router": {
       "accuracy": 0.48863636363636365,
       "cc": 33.92782003547969,
       "cc_by_type": {
        "choice": 21.96934509986067,
        "noul": 42.46575342465753,
        "score": 37.34836158192088
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 33.92782003547969
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3225806451612903,
       "cc": -20.34245261984394,
       "cc_by_type": {
        "choice": 16.923076923076923,
        "noul": -25,
        "score": -52.950434782608745
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.38412698412698415,
       "cc": 4.092899567611803,
       "cc_by_type": {
        "choice": 13.121361397223472,
        "noul": 8.045977011494255,
        "score": -8.888639705882317
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 4.092899567611803
      },
      "negation_pair_gen": {
       "accuracy": 0.39285714285714285,
       "cc": -26.347896825396816,
       "cc_by_type": {
        "choice": 9.523809523809527,
        "noul": 10.000000000000009,
        "score": -98.56749999999998
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.21052631578947367,
       "cc": 0.11098779134295962,
       "cc_by_type": {
        "choice": 0.11098779134295962
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 0.11098779134295962
      },
      "perturb_pair": {
       "accuracy": 0.5666666666666667,
       "cc": 48.77980199071942,
       "cc_by_type": {
        "choice": 25.29488859764089,
        "noul": 71.42857142857142,
        "score": 49.61594594594595
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 48.77980199071942
      },
      "usecase": {
       "accuracy": 0.3206751054852321,
       "cc": -11.511251481717963,
       "cc_by_type": {
        "choice": 6.340335324725405,
        "noul": -19.298245614035093,
        "score": -21.575844155844194
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.45454545454545453,
       "cc": -20.115,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": 0,
        "score": -80.345
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5,
       "cc": 49.152542372881356,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": 100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 49.152542372881356
      },
      "da": {
       "accuracy": 0.375,
       "cc": -13.013510848126225,
       "cc_by_type": {
        "choice": 28.994082840236683,
        "noul": -60,
        "score": -8.034615384615362
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.28888888888888886,
       "cc": -16.378460738952608,
       "cc_by_type": {
        "choice": 2.232951116475559,
        "noul": -33.333333333333336,
        "score": -18.035000000000046
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 0
      },
      "el": {
       "accuracy": 0.2222222222222222,
       "cc": -34.461847389558244,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -60,
        "score": -47.00000000000002
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.3708668453976765,
       "cc": 2.919018796085924,
       "cc_by_type": {
        "choice": 12.955692652832235,
        "noul": 2.2556390977443552,
        "score": -6.454275362318818
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 2.919018796085924
      },
      "es": {
       "accuracy": 0.42857142857142855,
       "cc": -0.37590909090909125,
       "cc_by_type": {
        "choice": 47.63636363636363,
        "noul": -27.27272727272727,
        "score": -21.491363636363637
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.4444444444444444,
       "cc": 47.183098591549296,
       "cc_by_type": {
        "choice": -5.63380281690141,
        "noul": 100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 47.183098591549296
      },
      "fr": {
       "accuracy": 0.4,
       "cc": -2.847552779456862,
       "cc_by_type": {
        "choice": 23.83638928067701,
        "noul": 0,
        "score": -32.3790476190476
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.4,
       "cc": 11.143196169772247,
       "cc_by_type": {
        "choice": 25.465838509316765,
        "noul": 39.99999999999999,
        "score": -32.03625000000001
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 11.143196169772247
      },
      "id": {
       "accuracy": 0.3333333333333333,
       "cc": 35.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 35.483870967741936
      },
      "it": {
       "accuracy": 0.5882352941176471,
       "cc": 39.10954484015044,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 50,
        "score": 18.104374999999983
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 39.10954484015044
      },
      "ja": {
       "accuracy": 0.28125,
       "cc": -2.907849641889275,
       "cc_by_type": {
        "choice": -13.821138211382111,
        "noul": 14.28571428571428,
        "score": -9.188124999999992
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.36363636363636365,
       "cc": 15.881704918032781,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 0,
        "score": -19.56800000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 15.881704918032781
      },
      "mixed": {
       "accuracy": 0.7272727272727273,
       "cc": 58.55131868131869,
       "cc_by_type": {
        "choice": 56.04395604395604,
        "noul": 50,
        "score": 69.61000000000004
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 58.55131868131869
      },
      "nl": {
       "accuracy": 0.38461538461538464,
       "cc": 5.939593933766626,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": 33.33333333333333,
        "score": -15.236000000000027
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 5.939593933766626
      },
      "no": {
       "accuracy": 0.7142857142857143,
       "cc": 67.21311475409837,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 67.21311475409837
      },
      "pl": {
       "accuracy": 0.475,
       "cc": 18.48718910116263,
       "cc_by_type": {
        "choice": 44.90599038041102,
        "noul": 39.99999999999999,
        "score": -29.444423076923123
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 18.48718910116263
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 18.251714405360136,
       "cc_by_type": {
        "choice": 45.7286432160804,
        "noul": 0,
        "score": 9.026500000000004
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 18.251714405360136
      },
      "sv": {
       "accuracy": 0.25,
       "cc": -12.537777777777777,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": 0,
        "score": -4.279999999999995
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.45454545454545453,
       "cc": 0.6930994152046814,
       "cc_by_type": {
        "choice": 21.052631578947373,
        "noul": -33.333333333333336,
        "score": 14.360000000000007
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0.6930994152046814
      },
      "uk": {
       "accuracy": 0.45454545454545453,
       "cc": -18.83422519509476,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": 30.453846153846154
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 47.33675675675678,
       "cc_by_type": {
        "choice": -29.729729729729726,
        "noul": 100,
        "score": 71.74000000000005
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 47.33675675675678
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.336,
       "cc": 13.974779754707205,
       "cc_by_type": {
        "choice": 13.974779754707205
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 13.974779754707205
      },
      "noul": {
       "accuracy": 0.528,
       "cc": 5.600000000000005,
       "cc_by_type": {
        "noul": 5.600000000000005
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 5.600000000000005
      },
      "score": {
       "accuracy": 0.32266666666666666,
       "cc": -8.548239929947377,
       "cc_by_type": {
        "score": -8.548239929947377
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2916666666666667,
       "cc": -53.64625846604429,
       "cc_by_type": {
        "choice": 11.037891268533771,
        "noul": 19.999999999999996,
        "score": -191.97666666666663
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -28.865504978177853,
       "cc_by_type": {
        "choice": -8.01963993453355,
        "noul": 19.999999999999996,
        "score": -98.576875
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.35,
       "cc": 2.0286683209263834,
       "cc_by_type": {
        "choice": -12.90322580645161,
        "noul": 25,
        "score": -6.010769230769242
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 2.0286683209263834
      },
      "demand_forecasting": {
       "accuracy": 0.38461538461538464,
       "cc": -8.96346355117439,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -50,
        "score": 19.495151515151534
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5,
       "cc": 24.71274167861124,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -19.999999999999996,
        "score": 50.03884615384615
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 24.71274167861124
      },
      "feature_extraction": {
       "accuracy": 0.27586206896551724,
       "cc": -17.316809116809114,
       "cc_by_type": {
        "choice": 5.9829059829059785,
        "noul": -19.999999999999996,
        "score": -37.93333333333333
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.46153846153846156,
       "cc": 3.7182956521739166,
       "cc_by_type": {
        "choice": 47.82608695652174,
        "noul": -25,
        "score": -11.671199999999992
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 3.7182956521739166
      },
      "gaming": {
       "accuracy": 0.27586206896551724,
       "cc": -26.63343815810153,
       "cc_by_type": {
        "choice": 4.950495049504946,
        "noul": -42.85714285714286,
        "score": -41.993666666666684
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.14285714285714285,
       "cc": -7.78137566137566,
       "cc_by_type": {
        "choice": -26.984126984126984,
        "noul": -60,
        "score": 63.64000000000001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.08695652173913043,
       "cc": -29.252194167155995,
       "cc_by_type": {
        "choice": -28.244274809160306,
        "noul": -19.999999999999996,
        "score": -39.512307692307694
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.36,
       "cc": 6.393062330623304,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": 0,
        "score": 0.47999999999999154
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 6.393062330623304
      },
      "legal_compliance": {
       "accuracy": 0.07692307692307693,
       "cc": -39.32074895531984,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100,
        "score": 13.185294117647029
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.4827586206896552,
       "cc": 11.789371912491177,
       "cc_by_type": {
        "choice": 17.431192660550458,
        "noul": 25,
        "score": -7.0630769230769275
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 11.789371912491177
      },
      "model_routing": {
       "accuracy": 0.5466666666666666,
       "cc": 35.63095556818022,
       "cc_by_type": {
        "choice": 35.674082015545444,
        "noul": 63.63636363636365,
        "score": 7.582421052631572
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 35.63095556818022
      },
      "moderation_trust_safety": {
       "accuracy": 0.25,
       "cc": -50.34215538847118,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100,
        "score": -87.86857142857143
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.6190476190476191,
       "cc": 44.91556070953055,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 66.66666666666667,
        "score": -1.7692307692307674
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 44.91556070953055
      },
      "risk_assessment": {
       "accuracy": 0.46153846153846156,
       "cc": -22.14570230607967,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": -33.333333333333336,
        "score": -76.50000000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.25,
       "cc": 0.1523784228166729,
       "cc_by_type": {
        "choice": -19.52191235059761,
        "noul": 14.28571428571428,
        "score": 5.69333333333335
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0.1523784228166729
      },
      "search_retrieval": {
       "accuracy": 0.36363636363636365,
       "cc": -15.631133289939976,
       "cc_by_type": {
        "choice": 14.08114558472553,
        "noul": 0,
        "score": -60.974545454545456
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      26.883939686902206,
      37.1769715121807
     ],
     "I": [
      3.840627882382685,
      11.691929733973469
     ],
     "capability": [
      16.023273303858527,
      23.5837139621214
     ],
     "composite": null,
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 32.62324971930184,
     "capability": 19.21888869011457,
     "intelligence": 5.814527660927306
    },
    "per_type": {
     "choice": {
      "accuracy": 0.336,
      "calibration": 14.023911671924271,
      "cc": 13.974779754707205,
      "cc_by_type": {
       "choice": 13.974779754707205
      },
      "cc_tiered": 14.761056872360381,
      "cc_tiered_open": 18.56541234955058,
      "cc_tiered_sealed": 10.956701395170182,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.974779754707205
     },
     "noul": {
      "accuracy": 0.528,
      "calibration": 33.12736263736261,
      "cc": 5.600000000000005,
      "cc_by_type": {
       "noul": 5.600000000000005
      },
      "cc_tiered": 2.6115347276674483,
      "cc_tiered_open": 1.9710144927536213,
      "cc_tiered_sealed": 3.2520549625812754,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.600000000000005
     },
     "score": {
      "accuracy": 0.32266666666666666,
      "calibration": 50.718474848618655,
      "cc": -8.548239929947377,
      "cc_by_type": {
       "score": -8.548239929947377
      },
      "cc_tiered": 0.07099138275408867,
      "cc_tiered_open": 0.14198276550817734,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 30.618919596314058,
     "capability": 18.126890524291774,
     "intelligence": 5.634861452269493
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.20870405610148388,
    "Emphasis on Accuracy 60:20:20": 0.1255993450091421,
    "Emphasis on Cost 20:20:60": 0.31061030752559765,
    "Emphasis on Speed 20:60:20": 0.31317253565694897,
    "Intelligence only": 0.0786327227711625
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)",
   "axes": {
    "calibration": 48.84618651494331,
    "cost": 79.50885516562305,
    "intelligence": 27.338167706853277,
    "speed": 91.03995916839054
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.28518548895899043,
      "ece_score": 42.96290220820191,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 42.96290220820191,
      "score_before_time_limit": 42.96290220820191,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.14010989010989006,
      "brier": 0.27496240302197805,
      "ece": 0.25560109890109894,
      "n": 364,
      "score": 48.87978021978021,
      "score_before_time_limit": 48.87978021978021,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.720108695652174,
      "interval90_width": 0.30865036231884047,
      "n": 368,
      "nrps": 0.18476832722826092,
      "score": 54.69587711684781,
      "score_before_time_limit": 54.69587711684781,
      "time_limit_failed_share": 0,
      "top_ece": 0.3606570652173914
     }
    },
    "score": 48.84618651494331
   },
   "capability": 38.09217711089829,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 735,765 input + 0 output tokens over 1,500 decisions = 490.51 tokens/decision. GPU-time cost USD 0.00481990147894455/1,000; old list-price cost USD 0.0147153/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00481990147894455/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004819901478944551
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ClassOne Qwen 3.5 4B (Qwen3.5-4B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "The author's own README Quickstart install line (pip install -e '.[dev]') at code pin github.com/devops-thiago/class-one@6cca132b and their own serve line (uvicorn classone.server.app:app); their server registers POST /v1/systemone itself, so the request and answer shapes are theirs. CLASSONE_BASE_MODEL was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, which their own code accepts, so the revision cannot drift to main and classone_heads.pt resolves locally with no request-time hub fetch. Their hard-coded MAX_TOKENS_PER_REQUEST = 8192 refuses longer prompts with HTTP 400; those items are recorded as invalid, the same way for every system. ONE documented mapping of ours and the only one: their server keys score probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1', so run 58's byte-identical ClassOneReindexAdapter relabels score keys k -> k-1 by calling the unchanged vendored typesafe adapter's own run(), and an item whose key set is not exactly {'1'..'n'} is REFUSED rather than rescored on a guess; choice and noul are untouched. The two already-published ClassOne rows carry exactly this treatment. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 36.145599118617284,
    "I_sealed": 18.530736295089266,
    "base": 27.338167706853277,
    "excess": 7.541618242188305,
    "gap": 17.614862823528018,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 59.400926298363274,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 64.97297297297298,
       "hard": 41.49103491664046,
       "judge": 64.40342781806197,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 24.24913733609386,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 13.043478260869556,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 24.786733721394732,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 39.40090909090911,
       "hard": 29.234259259259243,
       "judge": 20.891525423728808,
       "standard": 14.4274074074074
      }
     },
     "sealed|choice": {
      "cc": 38.6857940545428,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 45.56262383981645,
       "hard": 36.68242277673958,
       "judge": 33.33671553954645,
       "standard": 47.27773949000691
      }
     },
     "sealed|noul": {
      "cc": 16.457796589375537,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 38.46153846153846,
       "hard": -4.545454545454541,
       "judge": 30.555555555555557,
       "standard": 26.315789473684205
      }
     },
     "sealed|score": {
      "cc": 0.4486182413494726,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 28.812056074766346,
       "hard": -10.255555555555485,
       "judge": -2.3791150442477926,
       "standard": 11.916846846846852
      }
     }
    }
   },
   "jevbench_score": 14.833894246342627,
   "key": "classone-qwen3.5-4b",
   "last_measured_on": "2026-10-10",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "devops-thiago/classone-qwen3.5-4b@6626fdea7a3e8786bad8d2d5cc8acded1756a17c",
   "noul_decisive": {
    "acc_among_decisive": 0.7092651757188498,
    "decisive": 313,
    "decisive_rate": 0.8598901098901099,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 48.84618651494331,
     "cost": 64.96692638610241,
     "intelligence": 27.338167706853277,
     "speed": 91.46618934008248
    },
    "board_release": "v1.6.43",
    "capability": 38.09217711089829,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 116,
    "capability_rank": 144,
    "composite_ci95": {
     "A": [
      7.891109860006282,
      20.334524220090714
     ],
     "B": [
      6.6193128804538945,
      17.999509630335773
     ],
     "C": [
      5.479937402782141,
      14.121197375063
     ]
    },
    "jevbench_score": 14.342131800224653,
    "measured_in": "v1.6.9",
    "method": "v1.6",
    "open_board_rank": 122,
    "rank": 161,
    "ranks": {
     "A": 161,
     "B": 161,
     "C": 162,
     "capability": 208
    },
    "scores": {
     "A": 14.342131800224653,
     "B": 12.460860262475673,
     "C": 9.959813750156009
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23571948995813727,
     "p50_s_raw": 0.04285974497906864,
     "p95_s_adjusted": 0.3026812879485078,
     "p95_s_raw": 0.07634064397425391
    },
    "views": {
     "Balanced 33:33:33": 14.257369523966407,
     "Emphasis on Accuracy 60:20:20": 10.98579554132928,
     "Emphasis on Cost 20:20:60": 15.954348902584416,
     "Emphasis on Speed 20:60:20": 17.632944190866688,
     "Intelligence only": 8.172749758440709
    }
   },
   "rank": 206,
   "ranked": true,
   "ranks": {
    "A": 206,
    "B": 207,
    "C": 206,
    "capability": 250
   },
   "repo": "https://huggingface.co/devops-thiago/classone-qwen3.5-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 14.833894246342627,
    "B": 12.754758322548588,
    "C": 10.301315448849047
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2453273990890011,
    "p50_s_raw": 0.04766369954450056,
    "p95_s_adjusted": 0.3208174840314314,
    "p95_s_raw": 0.08540874201571569
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6206896551724138,
       "cc": 43.73972222222221,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 33.33333333333333,
        "score": 39.55249999999998
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 43.73972222222221
      },
      "calibration_gen": {
       "accuracy": 0.38392857142857145,
       "cc": -2.796751473164511,
       "cc_by_type": {
        "choice": 30.124223602484474,
        "noul": 7.692307692307687,
        "score": -46.206785714285694
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.5510204081632653,
       "cc": 31.21112378747796,
       "cc_by_type": {
        "choice": 48.14814814814815,
        "noul": 35.71428571428572,
        "score": 9.770937499999999
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 31.21112378747796
      },
      "fairness": {
       "accuracy": 0.7777777777777778,
       "cc": 37.86533333333333,
       "cc_by_type": {
        "choice": 100,
        "noul": -66.66666666666667,
        "score": 80.26266666666666
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 37.86533333333333
      },
      "jaggedness": {
       "accuracy": 0.42702702702702705,
       "cc": 3.439985798532349,
       "cc_by_type": {
        "choice": 37.47105141269106,
        "noul": -38.46153846153846,
        "score": 11.310444444444446
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 3.439985798532349
      },
      "llm_router": {
       "accuracy": 0.6628787878787878,
       "cc": 56.02907910025096,
       "cc_by_type": {
        "choice": 55.411054342777526,
        "noul": 53.42465753424656,
        "score": 59.2515254237288
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 56.02907910025096
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5161290322580645,
       "cc": 15.55809364548495,
       "cc_by_type": {
        "choice": 26.153846153846153,
        "noul": 50,
        "score": -29.4795652173913
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 15.55809364548495
      },
      "multilingual": {
       "accuracy": 0.5015873015873016,
       "cc": 20.949561234629694,
       "cc_by_type": {
        "choice": 36.40841916703987,
        "noul": 28.735632183908045,
        "score": -2.2953676470588213
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 20.949561234629694
      },
      "negation_pair_gen": {
       "accuracy": 0.42857142857142855,
       "cc": -45.491190476190475,
       "cc_by_type": {
        "choice": 28.571428571428573,
        "noul": 0,
        "score": -165.04500000000002
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.6666666666666666,
       "cc": 57.82463928967813,
       "cc_by_type": {
        "choice": 57.82463928967813
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 57.82463928967813
      },
      "perturb_pair": {
       "accuracy": 0.8166666666666667,
       "cc": 78.77666619437262,
       "cc_by_type": {
        "choice": 72.47706422018348,
        "noul": 85.71428571428572,
        "score": 78.13864864864864
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 78.77666619437262
      },
      "usecase": {
       "accuracy": 0.459915611814346,
       "cc": 13.14458872834252,
       "cc_by_type": {
        "choice": 31.778762767392575,
        "noul": 8.771929824561408,
        "score": -1.1169264069264218
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 13.14458872834252
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6363636363636364,
       "cc": 13.13722222222224,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": 50,
        "score": -57.254999999999946
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 13.13722222222224
      },
      "cs": {
       "accuracy": 0.5,
       "cc": 7.90960451977401,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 7.90960451977401
      },
      "da": {
       "accuracy": 0.5416666666666666,
       "cc": 36.62549309664694,
       "cc_by_type": {
        "choice": 40.8284023668639,
        "noul": 100,
        "score": -30.951923076923073
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 36.62549309664694
      },
      "de": {
       "accuracy": 0.5111111111111111,
       "cc": 22.671439359983808,
       "cc_by_type": {
        "choice": 40.25347012673505,
        "noul": 11.111111111111116,
        "score": 16.649736842105256
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 22.671439359983808
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": -25.492449799196788,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -19.999999999999996,
        "score": -72.14
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.5192135835567471,
       "cc": 21.206639325438474,
       "cc_by_type": {
        "choice": 43.690409422321885,
        "noul": 9.774436090225569,
        "score": 10.155072463767967
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 21.206639325438474
      },
      "es": {
       "accuracy": 0.7142857142857143,
       "cc": 55.06090909090909,
       "cc_by_type": {
        "choice": 47.63636363636363,
        "noul": 100,
        "score": 17.546363636363637
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 55.06090909090909
      },
      "fi": {
       "accuracy": 0.4444444444444444,
       "cc": 24.4131455399061,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 24.4131455399061
      },
      "fr": {
       "accuracy": 0.6,
       "cc": 26.589757539122846,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 0,
        "score": 5.15714285714286
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 26.589757539122846
      },
      "hi": {
       "accuracy": 0.4,
       "cc": 9.455742753623186,
       "cc_by_type": {
        "choice": 13.043478260869566,
        "noul": 19.999999999999996,
        "score": -4.676250000000004
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 9.455742753623186
      },
      "id": {
       "accuracy": 0.16666666666666666,
       "cc": -14.516129032258062,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7647058823529411,
       "cc": 59.36320992007523,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 50,
        "score": 53.4775
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 59.36320992007523
      },
      "ja": {
       "accuracy": 0.3125,
       "cc": 1.1244367015098795,
       "cc_by_type": {
        "choice": 2.4390243902439046,
        "noul": -14.28571428571429,
        "score": 15.220000000000022
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 1.1244367015098795
      },
      "ko": {
       "accuracy": 0.7272727272727273,
       "cc": 77.836,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 33.507999999999996
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 77.836
      },
      "mixed": {
       "accuracy": 1,
       "cc": 98.28333333333332,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 94.84999999999997
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 98.28333333333332
      },
      "nl": {
       "accuracy": 0.3076923076923077,
       "cc": -7.494183844011142,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": 0,
        "score": -22.204
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.5714285714285714,
       "cc": 33.87978142076503,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 33.87978142076503
      },
      "pl": {
       "accuracy": 0.575,
       "cc": 30.731736262935428,
       "cc_by_type": {
        "choice": 68.51770878880629,
        "noul": 60.00000000000001,
        "score": -36.32250000000001
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 30.731736262935428
      },
      "pt": {
       "accuracy": 0.7222222222222222,
       "cc": 60.06590480178671,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": 66.66666666666667,
        "score": 31.62150000000001
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 60.06590480178671
      },
      "sv": {
       "accuracy": 0.5,
       "cc": 0.943703703703702,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 50,
        "score": -58.28
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0.943703703703702
      },
      "tr": {
       "accuracy": 0.45454545454545453,
       "cc": 3.5850292397660724,
       "cc_by_type": {
        "choice": 47.368421052631575,
        "noul": 33.33333333333333,
        "score": -69.94666666666669
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 3.5850292397660724
      },
      "uk": {
       "accuracy": 0.09090909090909091,
       "cc": -59.95774804905239,
       "cc_by_type": {
        "choice": -30.43478260869565,
        "noul": -100,
        "score": -49.43846153846154
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 1,
       "cc": 69.91333333333333,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 9.739999999999993
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 69.91333333333333
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5573333333333333,
       "cc": 42.6498531698048,
       "cc_by_type": {
        "choice": 42.6498531698048
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 42.6498531698048
      },
      "noul": {
       "accuracy": 0.592,
       "cc": 18.399999999999995,
       "cc_by_type": {
        "noul": 18.399999999999995
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 18.399999999999995
      },
      "score": {
       "accuracy": 0.384,
       "cc": 6.636094570928175,
       "cc_by_type": {
        "score": 6.636094570928175
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 6.636094570928175
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4166666666666667,
       "cc": -33.318480688266526,
       "cc_by_type": {
        "choice": 11.037891268533771,
        "noul": 19.999999999999996,
        "score": -130.99333333333334
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": 0.8938591107473973,
       "cc_by_type": {
        "choice": 1.8003273322422266,
        "noul": 60.00000000000001,
        "score": -59.11875000000004
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0.8938591107473973
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 25.121935483870967,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": 25,
        "score": 47.13999999999999
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 25.121935483870967
      },
      "demand_forecasting": {
       "accuracy": 0.34615384615384615,
       "cc": 2.214011196300356,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": 0,
        "score": 3.0275757575757734
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 2.214011196300356
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4444444444444444,
       "cc": 10.85620321707278,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -19.999999999999996,
        "score": 8.469230769230762
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 10.85620321707278
      },
      "feature_extraction": {
       "accuracy": 0.27586206896551724,
       "cc": -14.564156954156951,
       "cc_by_type": {
        "choice": 23.076923076923077,
        "noul": -60,
        "score": -6.769393939393931
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.38461538461538464,
       "cc": 6.2751362318840584,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0,
        "score": -15.957200000000004
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 6.2751362318840584
      },
      "gaming": {
       "accuracy": 0.4482758620689655,
       "cc": 11.478893603646071,
       "cc_by_type": {
        "choice": 28.712871287128706,
        "noul": 42.85714285714286,
        "score": -37.133333333333354
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 11.478893603646071
      },
      "insurance_claims": {
       "accuracy": 0.2857142857142857,
       "cc": -53.55068783068785,
       "cc_by_type": {
        "choice": 36.507936507936506,
        "noul": -100,
        "score": -97.16000000000005
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.43478260869565216,
       "cc": 7.427903699354076,
       "cc_by_type": {
        "choice": 26.717557251908392,
        "noul": 19.999999999999996,
        "score": -24.43384615384616
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 7.427903699354076
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 20.725170731707326,
       "cc_by_type": {
        "choice": 51.21951219512196,
        "noul": 25,
        "score": -14.043999999999969
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 20.725170731707326
      },
      "legal_compliance": {
       "accuracy": 0.15384615384615385,
       "cc": -29.474503375120538,
       "cc_by_type": {
        "choice": -14.754098360655735,
        "noul": -100,
        "score": 26.330588235294115
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5172413793103449,
       "cc": 22.654965890378737,
       "cc_by_type": {
        "choice": 44.95412844036697,
        "noul": 25,
        "score": -1.9892307692307654
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 22.654965890378737
      },
      "model_routing": {
       "accuracy": 0.7333333333333333,
       "cc": 62.21502323863556,
       "cc_by_type": {
        "choice": 67.83704100777273,
        "noul": 68.18181818181819,
        "score": 50.626210526315795
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 62.21502323863556
      },
      "moderation_trust_safety": {
       "accuracy": 0.6875,
       "cc": 29.053859649122796,
       "cc_by_type": {
        "choice": 52.63157894736842,
        "noul": 100,
        "score": -65.47000000000003
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 29.053859649122796
      },
      "recruiting": {
       "accuracy": 0.7142857142857143,
       "cc": 37.555988059957905,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 33.33333333333333,
        "score": 9.485384615384618
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 37.555988059957905
      },
      "risk_assessment": {
       "accuracy": 0.46153846153846156,
       "cc": -1.4943134171907815,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": 33.33333333333333,
        "score": -81.21250000000002
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.40625,
       "cc": 17.67869126667931,
       "cc_by_type": {
        "choice": 12.350597609561751,
        "noul": 42.85714285714286,
        "score": -2.171666666666683
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 17.67869126667931
      },
      "search_retrieval": {
       "accuracy": 0.5909090909090909,
       "cc": 36.62560810973699,
       "cc_by_type": {
        "choice": 42.72076372315036,
        "noul": 33.33333333333333,
        "score": 33.82272727272727
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 36.62560810973699
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      43.368446520267796,
      53.193644571845724
     ],
     "I": [
      21.59802304927416,
      31.44548663789592
     ],
     "capability": [
      33.4245046846555,
      41.60857704511476
     ],
     "composite": null,
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 48.84618651494331,
     "capability": 38.09217711089829,
     "intelligence": 27.338167706853277
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5573333333333333,
      "calibration": 42.96290220820191,
      "cc": 42.6498531698048,
      "cc_by_type": {
       "choice": 42.6498531698048
      },
      "cc_tiered": 49.043360176453035,
      "cc_tiered_open": 59.400926298363274,
      "cc_tiered_sealed": 38.6857940545428,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 42.6498531698048
     },
     "noul": {
      "accuracy": 0.592,
      "calibration": 48.87978021978021,
      "cc": 18.399999999999995,
      "cc_by_type": {
       "noul": 18.399999999999995
      },
      "cc_tiered": 20.3534669627347,
      "cc_tiered_open": 24.24913733609386,
      "cc_tiered_sealed": 16.457796589375537,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.399999999999995
     },
     "score": {
      "accuracy": 0.384,
      "calibration": 54.69587711684781,
      "cc": 6.636094570928175,
      "cc_by_type": {
       "score": 6.636094570928175
      },
      "cc_tiered": 12.617675981372102,
      "cc_tiered_open": 24.786733721394732,
      "cc_tiered_sealed": 0.4486182413494726,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 6.636094570928175
     }
    },
    "raw_on_A": {
     "calibration": 44.37411603270306,
     "capability": 35.50100696428197,
     "intelligence": 26.62789789586089
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.9126377087926,
    "Emphasis on Accuracy 60:20:20": 11.213595080644025,
    "Emphasis on Cost 20:20:60": 17.524539007484908,
    "Emphasis on Speed 20:60:20": 18.20454210361668,
    "Intelligence only": 8.172749758440709
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Thiago Silva / @devops_thiago (thi.winnt@gmail.com)",
   "axes": {
    "calibration": 58.71667533779978,
    "cost": 68.99811555861743,
    "intelligence": 42.225671849648506,
    "speed": 89.69784178772274
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.24400358306188902,
      "ece_score": 51.199283387622195,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 51.199283387622195,
      "score_before_time_limit": 51.199283387622195,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.03234501347708896,
      "brier": 0.20490283425876044,
      "ece": 0.20667035040431214,
      "n": 371,
      "score": 58.66592991913757,
      "score_before_time_limit": 58.66592991913757,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.7696476964769647,
      "interval90_width": 0.23373983739837403,
      "n": 369,
      "nrps": 0.141136266192412,
      "score": 66.2848127066396,
      "score_before_time_limit": 66.2848127066396,
      "time_limit_failed_share": 0,
      "top_ece": 0.266583739837398
     }
    },
    "score": 58.71667533779978
   },
   "capability": 50.471173593724146,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 721,446 input + 0 output tokens over 1,477 decisions = 488.453622207 tokens/decision. GPU-time cost USD 0.0107993134822174/1,000; old list-price cost USD 0.0488453622207177/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0107993134822174/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.0480964
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01079931348221736
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ClassOne Qwen 3.5 9B (Qwen3.5-9B backbone with ClassOne decision heads, schema-constrained single forward pass, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...). Weights devops-thiago/classone-qwen3.5-9b (sha a45d16f1f5d1840a420f8ede59fa5cca3f079961), Apache-2.0, public and ungated; code github.com/devops-thiago/class-one, Apache-2.0. ONE DOCUMENTED MAPPING OF OURS, AND IT IS THE ONLY ONE. The author's server keys SCORE probabilities '1'..'n' while JevBench's canonical keys are '0'..'n-1'. The UNAMENDED smoke gate REFUSED the first attempt at check score-keys-are-indices, before any sealed item was sent, and the server was torn down with the card verified free. The 1-based convention - which their own schemas.py documents only as 'Probability distribution across each ordered rubric level', without stating the convention - was then established on the AUTHOR'S OWN SERVER with SYNTHETIC requests across every rubric size their own validator allows (2..10): exactly the keys '1'..'n' in order every time, never '0' and never a key above n. It is corroborated independently by their own `score` field being a 1-based probability-weighted mean of the same distribution (1.0185 on a 2-level rubric, 1.6147 on a 10-level one, 3.1155 where the mass sits on key 3; a 0-based convention would have given 0.0185), and a reversed rubric moved the mass to a different key, so 'k' means the k-th level of the criteria array AS SENT - which is what JevBench's 'k-1' means. classone_reindex_r58.ClassOneReindexAdapter therefore relabels score keys k -> k-1 and NOTHING else: the request body, the answer mapping, the noul range check and the choice-must-be-one-of-the-item's-labels rule are the UNCHANGED vendored typesafe adapter's, reused by calling its own run(). An item whose score key set is not exactly {'1'..'n'} is REFUSED and recorded invalid rather than rescored on a guess; none was. choice and noul are untouched. The gate then ran AMENDED in exactly that one check, requiring the key set to be exactly {'1'..'n'} - stricter than the original, because it also pins the count - with the other seven byte-identical, and passed. PREREG-R58-AMENDMENT-1.md (sha256 in PREREG-R58-AMENDMENT-1.md.sha256), receipt PROBE-classone-e2b.json. No file of the author's was edited and no prompt of ours exists. THE SUBMIT FORM CARRIED NO DESCRIPTION FOR EITHER CLASSONE ROW, so everything about how it is run comes from the author's own repository: install line README Quickstart step 1 (`pip install -e \".[dev]\"`, in an isolated venv - torch 2.14.1+cu130, transformers 5.19.0, fastapi 0.142.2), serve line step 3 (`uvicorn classone.server.app:app`), and the checkpoint selected the way their own src/classone/server/app.py reads it (CLASSONE_BASE_MODEL). THEIR SERVER REGISTERS POST /v1/systemone ITSELF (app.py:115, next to /v1/decide and /v1/classone) and their DecisionRequest/DecisionResponse in src/classone/schemas.py are the TypeSafe System One shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. output_tokens is 0 by their own schema comment. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. add-requests run 58, benchmarkheaven.com/submit ref 957CEE0B (bh-submit b45cdc66-df44-4d77-a616-e7c67c83706d)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "shapes, so no transport shim exists anywhere in the path. One RTX 5090 32 GB",
   "intelligence": {
    "I_open": 42.822420470216805,
    "I_sealed": 41.628923229080215,
    "base": 42.225671849648506,
    "excess": -1.3798904013016533,
    "gap": 1.1934972411365905,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 54.893255386978964,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 49.040578798364265,
       "judge": 36.71720500988794,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 50.0351966873706,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 23.538809336300865,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.58363636363636,
       "hard": 33.53250000000001,
       "judge": 30.56728813559323,
       "standard": -19.51370370370371
      }
     },
     "sealed|choice": {
      "cc": 50.4691947514562,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 39.42307692307691,
       "judge": 51.58286778398511,
       "standard": 63.419140288212844
      }
     },
     "sealed|noul": {
      "cc": 45.96319747817513,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72,
       "hard": 20.388349514563096,
       "judge": 57.89473684210527,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 28.454377457609333,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 65.85865248226952,
       "hard": 11.203235294117707,
       "judge": 44.33532934131734,
       "standard": 20.433096446700493
      }
     }
    }
   },
   "jevbench_score": 42.993233043857074,
   "key": "classone-qwen3.5-9b",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "devops-thiago/classone-qwen3.5-9b@a45d16f1f5d1840a420f8ede59fa5cca3f079961",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7855153203342619,
    "decisive_rate": 0.967654986522911,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 58.71667533779978,
     "cost": 49.33529996402755,
     "intelligence": 42.225671849648506,
     "speed": 90.24509258883624
    },
    "board_release": "v1.6.43",
    "capability": 50.471173593724146,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 90,
    "capability_rank": 117,
    "composite_ci95": {
     "A": [
      24.985716007949218,
      50.94397024296676
     ],
     "B": [
      22.880770578118387,
      48.816339954765404
     ],
     "C": [
      17.35119167218696,
      35.37775711317136
     ]
    },
    "jevbench_score": 38.54185769307265,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 55,
    "rank": 85,
    "ranks": {
     "A": 85,
     "B": 85,
     "C": 86,
     "capability": 172
    },
    "scores": {
     "A": 38.54185769307265,
     "B": 36.26089754980279,
     "C": 26.765178953522675
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2636520860018209,
     "p50_s_raw": 0.05682604300091043,
     "p95_s_adjusted": 0.35847554139327253,
     "p95_s_raw": 0.10423777069663628
    },
    "views": {
     "Balanced 33:33:33": 37.852042064412586,
     "Emphasis on Accuracy 60:20:20": 33.90552600237263,
     "Emphasis on Cost 20:20:60": 36.32702436923972,
     "Emphasis on Speed 20:60:20": 44.975093090995614,
     "Intelligence only": 29.320085596648227
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 141,
   "ranked": true,
   "ranks": {
    "A": 141,
    "B": 139,
    "C": 146,
    "capability": 210
   },
   "repo": "https://huggingface.co/devops-thiago/classone-qwen3.5-9b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 42.993233043857074,
    "B": 39.60602503432449,
    "C": 29.856411836011862
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27806573198759,
    "p50_s_raw": 0.06403286599379499,
    "p95_s_adjusted": 0.3855390299024293,
    "p95_s_raw": 0.11776951495121465
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-9B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      54.01204599443065,
      63.223240175665964
     ],
     "I": [
      35.33390022059835,
      47.469588715973586
     ],
     "capability": [
      45.37052539208047,
      54.754194446307686
     ],
     "composite": null,
     "seed": 27
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 58.71667533779978,
     "capability": 50.471173593724146,
     "intelligence": 42.225671849648506
    },
    "per_type": {
     "choice": {
      "accuracy": 0.624,
      "calibration": 51.199283387622195,
      "cc": 51.9149702073643,
      "cc_by_type": {
       "choice": 51.9149702073643
      },
      "cc_tiered": 52.68122506921758,
      "cc_tiered_open": 54.893255386978964,
      "cc_tiered_sealed": 50.4691947514562,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 51.9149702073643
     },
     "noul": {
      "accuracy": 0.752,
      "calibration": 58.66592991913757,
      "cc": 50.4,
      "cc_by_type": {
       "noul": 50.4
      },
      "cc_tiered": 47.99919708277287,
      "cc_tiered_open": 50.0351966873706,
      "cc_tiered_sealed": 45.96319747817513,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 50.4
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 66.2848127066396,
      "cc": 27.883929824561417,
      "cc_by_type": {
       "score": 27.883929824561417
      },
      "cc_tiered": 25.9965933969551,
      "cc_tiered_open": 23.538809336300865,
      "cc_tiered_sealed": 28.454377457609333,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.883929824561417
     }
    },
    "raw_on_A": {
     "calibration": 63.67698210538227,
     "capability": 55.155564978213434,
     "intelligence": 46.63414785104459
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 43.37868662775133,
    "Emphasis on Accuracy 60:20:20": 36.88147104343812,
    "Emphasis on Cost 20:20:60": 45.537002864039145,
    "Emphasis on Speed 20:60:20": 49.79003983908279,
    "Intelligence only": 30.115473528822495
   }
  },
  {
   "adapter_id": "jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "alt": {
    "axes": {
     "calibration": 86.72295444340945,
     "cost": 35.393298680703595,
     "intelligence": 67.9181689327805,
     "speed": 79.86099006822079
    },
    "label": "Workers AI price scenario (latency unmeasured)",
    "note": "Cloudflare standard launch input price; same self-hosted accuracy and adjusted H100 latency held fixed for this price-only scenario. No Workers AI latency was measured; this is not a hosted API result and does not set official Capability eligibility. Official self-hosted row uses the base-model reference price.",
    "pricing_only": true,
    "usd_per_1000": 0.1424153201970443
   },
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Cloudflare",
   "axes": {
    "calibration": 91.57703482429629,
    "cost": 55.47192130933339,
    "intelligence": 59.08165501447566,
    "speed": 82.80534424231135
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04191843750000011,
      "ece_score": 91.61631249999998,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 91.61631249999998,
      "score_before_time_limit": 91.61631249999998,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.18400000000000005,
      "brier": 0.08179747616000003,
      "ece": 0.035773866666666626,
      "n": 375,
      "score": 92.84522666666668,
      "score_before_time_limit": 92.84522666666668,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.92,
      "interval90_width": 0.31244444444444447,
      "n": 375,
      "nrps": 0.07581989387555554,
      "score": 90.26956530622223,
      "score_before_time_limit": 90.26956530622223,
      "time_limit_failed_share": 0,
      "top_ece": 0.05939439999999997
     }
    },
    "score": 91.57703482429629
   },
   "capability": 75.32934491938597,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 963,677 input + 0 output tokens over 1,624 decisions = 593.397167488 tokens/decision. GPU-time cost USD 0.0304978546082581/1,000; old list-price cost USD 0.249226810344828/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0304978546082581/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.030497854608258067
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Clef (Cloudflare, Qwen3.8-27B post-train with a joint schema head, multimodal, measured on text)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 57.04595985248707,
    "I_sealed": 61.117350176464235,
    "base": 59.08165501447566,
    "excess": -6.644777966415406,
    "gap": -4.0713903239771625,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 76.21786468827244,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 77.35136835482857,
       "judge": 72.31377719182596,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 43.36853002070393,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 30.000000000000004,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 51.55148484848486,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.974848484848486,
       "hard": 50.55657407407409,
       "judge": 67.49000000000001,
       "standard": 24.421851851851862
      }
     },
     "sealed|choice": {
      "cc": 74.36979440941545,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 69.9660633484163,
       "judge": 74.86033519553072,
       "standard": 78.93829289321346
      }
     },
     "sealed|noul": {
      "cc": 52.76571930304361,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72,
       "hard": 32.038834951456316,
       "judge": 63.1578947368421,
       "standard": 69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 56.216536816933655,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 56.72936170212767,
       "hard": 49.606985294117656,
       "judge": 66.35520958083832,
       "standard": 53.971218274111656
      }
     }
    }
   },
   "jevbench_score": 69.02603466604376,
   "key": "clef",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Cloudflare/clef@2f3de3dd85f379784083b0814d997ab627200f0c",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9411764705882353,
    "decisive_rate": 0.816,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.57703482429629,
     "cost": 28.102157220114762,
     "intelligence": 59.08165501447566,
     "speed": 83.3998209667487
    },
    "board_release": "v1.6.43",
    "capability": 75.32934491938597,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      16.278742772255967,
      17.01395343115317
     ],
     "B": [
      16.406079295395777,
      17.554401382209043
     ],
     "C": [
      13.014738345348354,
      17.01395343115317
     ]
    },
    "jevbench_score": 16.753692861922005,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 113,
    "rank": 150,
    "ranks": {
     "A": 150,
     "B": 140,
     "C": 131,
     "capability": 42
    },
    "scores": {
     "A": 16.753692861922005,
     "B": 17.103724421252515,
     "C": 16.244762021167798
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5763746980577708,
     "p50_s_raw": 0.21318734902888536,
     "p95_s_adjusted": 0.7930726912803948,
     "p95_s_raw": 0.3215363456401974
    },
    "views": {
     "Balanced 33:33:33": 14.692533500484354,
     "Emphasis on Accuracy 60:20:20": 16.05926983891725,
     "Emphasis on Cost 20:20:60": 11.641983750272722,
     "Emphasis on Speed 20:60:20": 17.850768289881103,
     "Intelligence only": 18.663451480355018
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 84,
   "ranked": true,
   "ranks": {
    "A": 84,
    "B": 84,
    "C": 71,
    "capability": 61
   },
   "repo": "https://huggingface.co/Cloudflare/clef",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.02603466604376,
    "B": 66.77807289481895,
    "C": 66.9292146905289
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.610504524409771,
    "p50_s_raw": 0.23025226220488548,
    "p95_s_adjusted": 0.8585719365626574,
    "p95_s_raw": 0.3542859682813287
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.28138326498332,
      91.94843594390738
     ],
     "I": [
      53.626530822718266,
      63.68048125704343
     ],
     "capability": [
      70.47069839174009,
      77.24671214753253
     ],
     "composite": null,
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.57703482429629,
     "capability": 75.32934491938597,
     "intelligence": 59.08165501447566
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8093333333333333,
      "calibration": 91.61631249999998,
      "cc": 75.61645652359255,
      "cc_by_type": {
       "choice": 75.61645652359255
      },
      "cc_tiered": 75.29382954884395,
      "cc_tiered_open": 76.21786468827244,
      "cc_tiered_sealed": 74.36979440941545,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.61645652359255
     },
     "noul": {
      "accuracy": 0.768,
      "calibration": 92.84522666666668,
      "cc": 53.6,
      "cc_by_type": {
       "noul": 53.6
      },
      "cc_tiered": 48.06712466187377,
      "cc_tiered_open": 43.36853002070393,
      "cc_tiered_sealed": 52.76571930304361,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 53.6
     },
     "score": {
      "accuracy": 0.6613333333333333,
      "calibration": 90.26956530622223,
      "cc": 54.4725614035088,
      "cc_by_type": {
       "score": 54.4725614035088
      },
      "cc_tiered": 53.88401083270926,
      "cc_tiered_open": 51.55148484848486,
      "cc_tiered_sealed": 56.216536816933655,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.4725614035088
     }
    },
    "raw_on_A": {
     "calibration": 88.38059707442306,
     "capability": 75.82665436300499,
     "intelligence": 63.27271165158693
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.789910141801414,
    "Emphasis on Accuracy 60:20:20": 61.819340910204524,
    "Emphasis on Cost 20:20:60": 60.18030653767556,
    "Emphasis on Speed 20:60:20": 70.2420708391423,
    "Intelligence only": 59.081655014475665
   }
  },
  {
   "adapter_id": "jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "alt": {
    "axes": {
     "calibration": 87.5533640278553,
     "cost": 48.17236064887202,
     "intelligence": 53.08161490373422,
     "speed": 83.63107135213085
    },
    "label": "Workers AI price scenario (latency unmeasured)",
    "note": "Cloudflare standard launch input price; same self-hosted accuracy and adjusted H100 latency held fixed for this price-only scenario. No Workers AI latency was measured; this is not a hosted API result and does not set official Capability eligibility. Official self-hosted row uses the base-model reference price.",
    "pricing_only": true,
    "usd_per_1000": 0.053405745073891626
   },
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Cloudflare",
   "axes": {
    "calibration": 85.46479352633332,
    "cost": 66.46245152664164,
    "intelligence": 42.18368888957246,
    "speed": 88.14167964175255
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1119512500000001,
      "ece_score": 77.60974999999998,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 77.60974999999998,
      "score_before_time_limit": 77.60974999999998,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.31733333333333336,
      "brier": 0.11674051933333338,
      "ece": 0.04965040000000006,
      "n": 375,
      "score": 90.06992,
      "score_before_time_limit": 90.06992,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9333333333333333,
      "interval90_width": 0.460888888888889,
      "n": 375,
      "nrps": 0.09493298842,
      "score": 88.71471057900001,
      "score_before_time_limit": 88.71471057900001,
      "time_limit_failed_share": 0,
      "top_ece": 0.06538639999999996
     }
    },
    "score": 85.46479352633332
   },
   "capability": 63.824241207952895,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 963,677 input + 0 output tokens over 1,624 decisions = 593.397167488 tokens/decision. GPU-time cost USD 0.0131195301658369/1,000; old list-price cost USD 0.0593397167487685/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0131195301658369/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013119530165836881
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Clef-Flash (Cloudflare, Qwen3.5-9B post-train with a joint schema head, multimodal, measured on text)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 45.25550449770323,
    "I_sealed": 39.11187328144169,
    "base": 42.18368888957246,
    "excess": 3.5702435738232956,
    "gap": 6.143631216261539,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 62.93462232292448,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 56.59012268008808,
       "judge": 68.35860250494396,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 30.42788129744652,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 30.434782608695656,
       "judge": 0,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 42.4040098727387,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 41.5230303030303,
       "hard": 37.280925925925935,
       "judge": 43.06322033898306,
       "standard": 52.101851851851855
      }
     },
     "sealed|choice": {
      "cc": 54.80150596378907,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 45.53167420814478,
       "judge": 53.44506517690876,
       "standard": 70.07020569035596
      }
     },
     "sealed|noul": {
      "cc": 20.824374761599753,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 52,
       "hard": -10.679611650485432,
       "judge": 31.578947368421062,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 41.70973911893625,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 54.71716312056738,
       "hard": 35.80301470588234,
       "judge": 41.72143712574849,
       "standard": 47.00192893401015
      }
     }
    }
   },
   "jevbench_score": 46.07172412904015,
   "key": "clef-flash",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Cloudflare/clef-flash@17f0b0ad64efb65d273590632833508766b2aae6",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.93359375,
    "decisive_rate": 0.6826666666666666,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.46479352633332,
     "cost": 46.79963593205177,
     "intelligence": 42.18368888957246,
     "speed": 88.4044196893156
    },
    "board_release": "v1.6.43",
    "capability": 63.824241207952895,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 41,
    "capability_rank": 64,
    "composite_ci95": {
     "A": [
      26.640037750072654,
      45.22280470107504
     ],
     "B": [
      24.191496498186186,
      42.57204972938278
     ],
     "C": [
      18.500026215328234,
      31.40472548685768
     ]
    },
    "jevbench_score": 36.635050479655426,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 60,
    "rank": 90,
    "ranks": {
     "A": 90,
     "B": 86,
     "C": 91,
     "capability": 106
    },
    "scores": {
     "A": 36.635050479655426,
     "B": 33.967249202727025,
     "C": 25.44100727753849
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3556603590026498,
     "p50_s_raw": 0.10283017950132489,
     "p95_s_adjusted": 0.405996760725975,
     "p95_s_raw": 0.12799838036298752
    },
    "views": {
     "Balanced 33:33:33": 33.178006784894365,
     "Emphasis on Accuracy 60:20:20": 30.03860559786673,
     "Emphasis on Cost 20:20:60": 31.455768602881705,
     "Emphasis on Speed 20:60:20": 39.463018165391404,
     "Intelligence only": 26.30501767147255
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 127,
   "ranked": true,
   "ranks": {
    "A": 127,
    "B": 133,
    "C": 132,
    "capability": 137
   },
   "repo": "https://huggingface.co/Cloudflare/clef-flash",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 46.07172412904015,
    "B": 41.62299336826614,
    "C": 31.99425286738899
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3573765229433775,
    "p50_s_raw": 0.10368826147168875,
    "p95_s_adjusted": 0.42924576271325354,
    "p95_s_raw": 0.13962288135662676
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-9B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.4975513933287,
      86.84610174167315
     ],
     "I": [
      36.95680839342918,
      46.203587775711
     ],
     "capability": [
      59.38252934298857,
      65.53439425442862
     ],
     "composite": null,
     "seed": 21
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.46479352633332,
     "capability": 63.824241207952895,
     "intelligence": 42.18368888957246
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6666666666666666,
      "calibration": 77.60974999999998,
      "cc": 57.37142748879814,
      "cc_by_type": {
       "choice": 57.37142748879814
      },
      "cc_tiered": 58.868064143356776,
      "cc_tiered_open": 62.93462232292448,
      "cc_tiered_sealed": 54.80150596378907,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.37142748879814
     },
     "noul": {
      "accuracy": 0.6373333333333333,
      "calibration": 90.06992,
      "cc": 27.46666666666666,
      "cc_by_type": {
       "noul": 27.46666666666666
      },
      "cc_tiered": 25.626128029523137,
      "cc_tiered_open": 30.42788129744652,
      "cc_tiered_sealed": 20.824374761599753,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 27.46666666666666
     },
     "score": {
      "accuracy": 0.6026666666666667,
      "calibration": 88.71471057900001,
      "cc": 42.01200877192982,
      "cc_by_type": {
       "score": 42.01200877192982
      },
      "cc_tiered": 42.05687449583748,
      "cc_tiered_open": 42.4040098727387,
      "cc_tiered_sealed": 41.70973911893625,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 42.01200877192982
     }
    },
    "raw_on_A": {
     "calibration": 82.15484704393162,
     "capability": 63.82820945816345,
     "intelligence": 45.501571872395274
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 42.624175536965765,
    "Emphasis on Accuracy 60:20:20": 36.498450151393136,
    "Emphasis on Cost 20:20:60": 44.381470276839956,
    "Emphasis on Speed 20:60:20": 48.894391547743,
    "Intelligence only": 30.02573569768127
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Cloudflare",
   "axes": {
    "calibration": 82.15469329918429,
    "cost": 56.02577392355937,
    "intelligence": 34.47716016566848,
    "speed": 86.57770472711044
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09821189024390234,
      "ece_score": 80.35762195121954,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 80.35762195121954,
      "score_before_time_limit": 80.35762195121954,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3493333333333334,
      "brier": 0.1572640221333334,
      "ece": 0.06652586666666661,
      "n": 375,
      "score": 86.69482666666669,
      "score_before_time_limit": 86.69482666666669,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8506666666666667,
      "interval90_width": 0.36177777777777775,
      "n": 375,
      "nrps": 0.11951350773999989,
      "score": 79.41163127966668,
      "score_before_time_limit": 79.41163127966668,
      "time_limit_failed_share": 0,
      "top_ece": 0.14612693333333326
     }
    },
    "score": 82.15469329918429
   },
   "capability": 58.315926732426384,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,755,730 input + 0 output tokens over 1,500 decisions = 1837.15333333 tokens/decision. GPU-time cost USD 0.0292285678292639/1,000; old list-price cost USD 0.275573/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0292285678292639/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02922856782926389
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Clef-omni",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX PRO 6000 Blackwell 96 GB pod (Beltsville US, 29 min, USD 1.19/h); the authors' own load_release_model() + systemone() from the pinned repo's joint_schema_model.py behind the run-39 loopback shim (127.0.0.1, single-threaded, no answer arithmetic), pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, defaults unchanged (max_length 64,000). Stack as the model card tested: torch 2.11.0+cu130, transformers 5.10.2, plus torchvision 0.26.0+cu130, which the Omni video processor requires to load. HF/transformers offline, no credentials, pod-level iptables egress cut installed and verified before the sealed input was uploaded. All 17 LFS files verified against the Hugging Face tree digests. GPU memory 62,729 MiB after load and at most 74,735 MiB during the run (per-minute log), of 97,887 MiB.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 37.25444425506104,
    "I_sealed": 31.699876076275928,
    "base": 34.47716016566848,
    "excess": -4.518676402554602,
    "gap": 5.554568178785111,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 65.02240890008018,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243245,
       "hard": 67.9144385026738,
       "judge": 56.49307844429796,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 10.814354727398204,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -13.043478260869568,
       "judge": 0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 35.92656913770473,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.82848484848485,
       "hard": 42.1251851851852,
       "judge": 49.07338983050846,
       "standard": -4.141851851851852
      }
     },
     "sealed|choice": {
      "cc": 53.547209374932514,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 62.45698195849411,
       "hard": 49.543805650214345,
       "judge": 52.513824767896104,
       "standard": 58.64920744314267
      }
     },
     "sealed|noul": {
      "cc": 9.67795362532205,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 43.58974358974359,
       "hard": -10.606060606060607,
       "judge": 16.666666666666675,
       "standard": 22.807017543859654
      }
     },
     "sealed|score": {
      "cc": 31.87446522857323,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 45.32728971962617,
       "hard": 22.845027777777847,
       "judge": 36.40668141592918,
       "standard": 36.408603603603595
      }
     }
    }
   },
   "jevbench_score": 26.94791782828204,
   "key": "clef-omni",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.6",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Cloudflare/clef-omni@0db1cd2607d76a7bdb2a382f659e7b313079f84b",
   "noul_decisive": {
    "acc_among_decisive": 0.8565573770491803,
    "decisive": 244,
    "decisive_rate": 0.6506666666666666,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.15469329918429,
     "cost": 26.792900075221908,
     "intelligence": 34.47716016566848,
     "speed": 86.7880661703893
    },
    "board_release": "v1.6.43",
    "capability": 58.315926732426384,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      3.8823605492743414,
      8.586118919617187
     ],
     "B": [
      3.562206076177268,
      8.318090708030555
     ],
     "C": [
      2.696083714773848,
      5.9625825830674914
     ]
    },
    "jevbench_score": 6.06640325953113,
    "measured_in": "v1.6.6",
    "method": "v1.6",
    "open_board_rank": 143,
    "rank": 183,
    "ranks": {
     "A": 183,
     "B": 182,
     "C": 184,
     "capability": 141
    },
    "scores": {
     "A": 6.06640325953113,
     "B": 5.735167687307061,
     "C": 4.2127800413410625
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.43618169901892545,
     "p50_s_raw": 0.14309084950946271,
     "p95_s_adjusted": 0.4803147715982049,
     "p95_s_raw": 0.16515738579910247
    },
    "views": {
     "Balanced 33:33:33": 5.261179060414329,
     "Emphasis on Accuracy 60:20:20": 5.02460048275502,
     "Emphasis on Cost 20:20:60": 4.476423118785528,
     "Emphasis on Speed 20:60:20": 6.765856602997514,
     "Intelligence only": 4.70710504931535
    }
   },
   "rank": 173,
   "ranked": true,
   "ranks": {
    "A": 173,
    "B": 171,
    "C": 175,
    "capability": 174
   },
   "repo": "https://huggingface.co/Cloudflare/clef-omni",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 26.94791782828204,
    "B": 23.873563968840394,
    "C": 18.71383182519586
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.44394626477733257,
    "p50_s_raw": 0.14697313238866627,
    "p95_s_adjusted": 0.49533511972986155,
    "p95_s_raw": 0.1726675598649308
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.896551724137931,
       "cc": 86.57097222222222,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 68.04625
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 86.57097222222222
      },
      "calibration_gen": {
       "accuracy": 0.5982142857142857,
       "cc": 32.91760471412644,
       "cc_by_type": {
        "choice": 55.745341614906835,
        "noul": 15.384615384615374,
        "score": 27.622857142857114
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 32.91760471412644
      },
      "composite_gen": {
       "accuracy": 0.6632653061224489,
       "cc": 50.659661596119925,
       "cc_by_type": {
        "choice": 74.07407407407408,
        "noul": 35.71428571428572,
        "score": 42.19062499999998
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 50.659661596119925
      },
      "fairness": {
       "accuracy": 0.7407407407407407,
       "cc": 51.41265359477125,
       "cc_by_type": {
        "choice": 88.23529411764707,
        "noul": 0,
        "score": 66.00266666666667
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 51.41265359477125
      },
      "jaggedness": {
       "accuracy": 0.42162162162162165,
       "cc": -0.8300322403138493,
       "cc_by_type": {
        "choice": 30.523390458545624,
        "noul": -53.84615384615385,
        "score": 20.832666666666675
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.6818181818181818,
       "cc": 57.872019428772546,
       "cc_by_type": {
        "choice": 50.76637250348352,
        "noul": 53.42465753424656,
        "score": 69.42502824858757
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 57.872019428772546
      },
      "long_state": {
       "accuracy": 0.3103448275862069,
       "cc": -1.3167821067821077,
       "cc_by_type": {
        "choice": 16.66666666666667,
        "noul": -27.27272727272727,
        "score": 6.655714285714276
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5483870967741935,
       "cc": 5.659007803790407,
       "cc_by_type": {
        "choice": 53.84615384615384,
        "noul": -25,
        "score": -11.869130434782615
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 5.659007803790407
      },
      "multilingual": {
       "accuracy": 0.5714285714285714,
       "cc": 29.241308106053285,
       "cc_by_type": {
        "choice": 54.32154052843709,
        "noul": 5.747126436781613,
        "score": 27.655257352941153
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 29.241308106053285
      },
      "negation_pair_gen": {
       "accuracy": 0.6607142857142857,
       "cc": 18.345555555555563,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 19.999999999999996,
        "score": -31.62999999999998
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 18.345555555555563
      },
      "option_order": {
       "accuracy": 0.8947368421052632,
       "cc": 86.68146503884572,
       "cc_by_type": {
        "choice": 86.68146503884572
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 86.68146503884572
      },
      "perturb_pair": {
       "accuracy": 0.9,
       "cc": 81.10219498659866,
       "cc_by_type": {
        "choice": 84.27260812581913,
        "noul": 71.42857142857142,
        "score": 87.6054054054054
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 81.10219498659866
      },
      "usecase": {
       "accuracy": 0.5232067510548524,
       "cc": 17.11602027853846,
       "cc_by_type": {
        "choice": 53.74831374060514,
        "noul": -15.789473684210531,
        "score": 13.389220779220778
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 17.11602027853846
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.45454545454545453,
       "cc": -23.164444444444445,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": -50,
        "score": -66.16
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5,
       "cc": 7.90960451977401,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 7.90960451977401
      },
      "da": {
       "accuracy": 0.5416666666666666,
       "cc": 23.599881656804737,
       "cc_by_type": {
        "choice": 52.662721893491124,
        "noul": -19.999999999999996,
        "score": 38.13692307692308
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 23.599881656804737
      },
      "de": {
       "accuracy": 0.7111111111111111,
       "cc": 51.80671574665221,
       "cc_by_type": {
        "choice": 72.84248642124321,
        "noul": 55.55555555555556,
        "score": 27.02210526315789
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 51.80671574665221
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": -55.7336546184739,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": -100,
        "score": -106.96
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.6094727435210009,
       "cc": 34.60248155699925,
       "cc_by_type": {
        "choice": 56.70218732473357,
        "noul": 9.774436090225569,
        "score": 37.33082125603862
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 34.60248155699925
      },
      "es": {
       "accuracy": 0.7142857142857143,
       "cc": 54.17545454545455,
       "cc_by_type": {
        "choice": 73.81818181818181,
        "noul": 63.63636363636365,
        "score": 25.071818181818184
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 54.17545454545455
      },
      "fi": {
       "accuracy": 0.1111111111111111,
       "cc": -52.816901408450704,
       "cc_by_type": {
        "choice": -5.63380281690141,
        "noul": -100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.6,
       "cc": 17.07420198356729,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": -50,
        "score": 26.610476190476174
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 17.07420198356729
      },
      "hi": {
       "accuracy": 0.6333333333333333,
       "cc": 44.937223084886135,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 60.00000000000001,
        "score": 12.078749999999994
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 44.937223084886135
      },
      "id": {
       "accuracy": 0.16666666666666666,
       "cc": -48.387096774193544,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7058823529411765,
       "cc": 50.644668253408554,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 50,
        "score": 27.321874999999995
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 50.644668253408554
      },
      "ja": {
       "accuracy": 0.46875,
       "cc": 18.159518002322883,
       "cc_by_type": {
        "choice": 26.829268292682926,
        "noul": -14.28571428571429,
        "score": 41.93500000000001
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 18.159518002322883
      },
      "ko": {
       "accuracy": 0.6363636363636364,
       "cc": 66.1930382513661,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": 31.365999999999982
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 66.1930382513661
      },
      "mixed": {
       "accuracy": 1,
       "cc": 85.82333333333332,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 57.469999999999956
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 85.82333333333332
      },
      "nl": {
       "accuracy": 0.5384615384615384,
       "cc": 15.964908077994421,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 0,
        "score": -1.9660000000000233
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 15.964908077994421
      },
      "no": {
       "accuracy": 0.2857142857142857,
       "cc": -15.846994535519126,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.575,
       "cc": 36.12173351608311,
       "cc_by_type": {
        "choice": 84.25885439440316,
        "noul": 19.999999999999996,
        "score": 4.106346153846174
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 36.12173351608311
      },
      "pt": {
       "accuracy": 0.8333333333333334,
       "cc": 74.19423813512005,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": 66.66666666666667,
        "score": 74.0065
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 74.19423813512005
      },
      "sv": {
       "accuracy": 0.375,
       "cc": 4.158888888888879,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": 50,
        "score": -4.190000000000027
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 4.158888888888879
      },
      "tr": {
       "accuracy": 0.7272727272727273,
       "cc": 22.35695906432748,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 33.33333333333333,
        "score": -39.94666666666669
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 22.35695906432748
      },
      "uk": {
       "accuracy": 0.7272727272727273,
       "cc": 12.17157190635451,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": -100,
        "score": 58.25384615384615
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 12.17157190635451
      },
      "zh": {
       "accuracy": 0.625,
       "cc": 50.52282282282281,
       "cc_by_type": {
        "choice": 35.13513513513514,
        "noul": 33.33333333333333,
        "score": 83.09999999999995
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 50.52282282282281
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6653333333333333,
       "cc": 56.64190706512352,
       "cc_by_type": {
        "choice": 56.64190706512352
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 56.64190706512352
      },
      "noul": {
       "accuracy": 0.5573333333333333,
       "cc": 11.466666666666669,
       "cc_by_type": {
        "noul": 11.466666666666669
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 11.466666666666669
      },
      "score": {
       "accuracy": 0.5333333333333333,
       "cc": 33.64992119089322,
       "cc_by_type": {
        "score": 33.64992119089322
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 33.64992119089322
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5416666666666666,
       "cc": 8.511753615229727,
       "cc_by_type": {
        "choice": 40.691927512355846,
        "noul": -60,
        "score": 44.843333333333334
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 8.511753615229727
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 9.050032733224224,
       "cc_by_type": {
        "choice": 70.54009819967267,
        "noul": -19.999999999999996,
        "score": -23.39
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 9.050032733224224
      },
      "customer_support": {
       "accuracy": 0.3,
       "cc": -13.059735318444998,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -75,
        "score": 0.336923076923068
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.38461538461538464,
       "cc": -8.969614214433495,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": -50,
        "score": -16.667878787878788
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5555555555555556,
       "cc": 47.39004937091894,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 60.00000000000001,
        "score": 38.07076923076923
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 47.39004937091894
      },
      "feature_extraction": {
       "accuracy": 0.4827586206896552,
       "cc": 12.539619269619266,
       "cc_by_type": {
        "choice": 48.71794871794871,
        "noul": -60,
        "score": 48.90090909090908
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 12.539619269619266
      },
      "financial_crime": {
       "accuracy": 0.5384615384615384,
       "cc": 14.042347826086965,
       "cc_by_type": {
        "choice": 73.91304347826087,
        "noul": 0,
        "score": -31.78599999999998
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 14.042347826086965
      },
      "gaming": {
       "accuracy": 0.41379310344827586,
       "cc": 5.618274556027025,
       "cc_by_type": {
        "choice": 28.712871287128706,
        "noul": -14.28571428571429,
        "score": 2.4276666666666613
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 5.618274556027025
      },
      "insurance_claims": {
       "accuracy": 0.2857142857142857,
       "cc": -11.208359788359795,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": -100,
        "score": 45.73999999999998
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.5217391304347826,
       "cc": 9.780414954002737,
       "cc_by_type": {
        "choice": 45.03816793893129,
        "noul": -60,
        "score": 44.30307692307692
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 9.780414954002737
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 5.368834688346889,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -75,
        "score": 7.3666666666666885
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 5.368834688346889
      },
      "legal_compliance": {
       "accuracy": 0.3076923076923077,
       "cc": -20.515570556091294,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": 4.027058823529394
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.6206896551724138,
       "cc": 32.95168666196189,
       "cc_by_type": {
        "choice": 63.30275229357799,
        "noul": 0,
        "score": 35.55230769230767
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 32.95168666196189
      },
      "model_routing": {
       "accuracy": 0.74,
       "cc": 63.43091944512483,
       "cc_by_type": {
        "choice": 69.44518895738408,
        "noul": 77.27272727272727,
        "score": 43.57484210526314
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 63.43091944512483
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 24.84468671679197,
       "cc_by_type": {
        "choice": 21.052631578947366,
        "noul": -19.999999999999996,
        "score": 73.48142857142854
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 24.84468671679197
      },
      "recruiting": {
       "accuracy": 0.6190476190476191,
       "cc": 25.69128720525705,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 0,
        "score": 7.2246153846153955
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 25.69128720525705
      },
      "risk_assessment": {
       "accuracy": 0.5384615384615384,
       "cc": 14.524019916142551,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": 33.33333333333333,
        "score": -33.15750000000002
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 14.524019916142551
      },
      "scientific_discovery": {
       "accuracy": 0.3125,
       "cc": -9.035004163241226,
       "cc_by_type": {
        "choice": 28.286852589641434,
        "noul": -42.85714285714286,
        "score": -12.53472222222225
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.8181818181818182,
       "cc": 63.02598587305031,
       "cc_by_type": {
        "choice": 71.36038186157518,
        "noul": 66.66666666666667,
        "score": 51.05090909090908
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 63.02598587305031
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.99498334146031,
      83.8879680508817
     ],
     "I": [
      28.570607529410697,
      40.0809890185093
     ],
     "capability": [
      53.56177237020907,
      61.07325750582405
     ],
     "composite": null,
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.15469329918429,
     "capability": 58.315926732426384,
     "intelligence": 34.47716016566848
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6653333333333333,
      "calibration": 80.35762195121954,
      "cc": 56.64190706512352,
      "cc_by_type": {
       "choice": 56.64190706512352
      },
      "cc_tiered": 59.28480913750634,
      "cc_tiered_open": 65.02240890008018,
      "cc_tiered_sealed": 53.547209374932514,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 56.64190706512352
     },
     "noul": {
      "accuracy": 0.5573333333333333,
      "calibration": 86.69482666666669,
      "cc": 11.466666666666669,
      "cc_by_type": {
       "noul": 11.466666666666669
      },
      "cc_tiered": 10.246154176360127,
      "cc_tiered_open": 10.814354727398204,
      "cc_tiered_sealed": 9.67795362532205,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 11.466666666666669
     },
     "score": {
      "accuracy": 0.5333333333333333,
      "calibration": 79.41163127966668,
      "cc": 33.64992119089322,
      "cc_by_type": {
       "score": 33.64992119089322
      },
      "cc_tiered": 33.90051718313898,
      "cc_tiered_open": 35.92656913770473,
      "cc_tiered_sealed": 31.87446522857323,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 33.64992119089322
     }
    },
    "raw_on_A": {
     "calibration": 80.93316891446167,
     "capability": 59.00548965209646,
     "intelligence": 37.07781038973123
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 24.423166106881276,
    "Emphasis on Accuracy 60:20:20": 20.42161555612092,
    "Emphasis on Cost 20:20:60": 25.26358560076144,
    "Emphasis on Speed 20:60:20": 29.168285359123715,
    "Intelligence only": 16.392849456500993
   }
  },
  {
   "adapter_id": "images/clm-8b/spec.json:clm_adapter.CLMAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Contrastive-LM (Kwok, Kang, Suresh, Saad-Falcon, Pavone, Ré, Mirhoseini)",
   "axes": {
    "calibration": 40.03295405037662,
    "cost": 73.74462936099019,
    "intelligence": 0.13075619159747265,
    "speed": 91.7219631344249
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.3993510766836583,
      "ece_score": 20.129784663268335,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 20.10294495038398,
      "score_before_time_limit": 20.129784663268335,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.4906666666666667,
      "brier": 0.26050031793956074,
      "ece": 0.18460455903608142,
      "n": 375,
      "score": 63.079088192783715,
      "score_before_time_limit": 63.079088192783715,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.688,
      "interval90_width": 0.5184444444444443,
      "n": 375,
      "nrps": 0.2846437031480146,
      "score": 36.91682900796216,
      "score_before_time_limit": 36.91682900796216,
      "time_limit_failed_share": 0,
      "top_ece": 0.48850985834637106
     }
    },
    "score": 40.03295405037662
   },
   "capability": 20.081855120987044,
   "class": "system-one-open",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 8B active parameters; throughput 30762.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 619,933 input + 0 output tokens over 1,624 decisions = 381.732142857 tokens/decision. GPU-time cost USD 0.00750203388276616/1,000; old list-price cost USD 0.0446626607142857/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00750203388276616/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.007502033882766164
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "CLM-8B (Contrastive-LM, clm-latest)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 0.2615123831949453,
    "I_sealed": 0,
    "base": 0.13075619159747265,
    "excess": -2.3118752592432985,
    "gap": 0.2615123831949453,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 0.7845371495848359,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -16.75675675675675,
       "hard": -7.5810003145643154,
       "judge": 16.941331575477918,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -56.52173913043479,
       "judge": -19.999999999999996,
       "standard": -44.44444444444444
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -21.94096032005719,
       "hard": -1.9559175706170606,
       "judge": -11.452540725941729,
       "standard": -95.63691251131768
      }
     },
     "sealed|choice": {
      "cc": 0,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -6.42228955731281,
       "hard": -4.355203619909522,
       "judge": 0.3724394785847514,
       "standard": 7.993595270353507
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -31.999999999999996,
       "hard": -33.98058252427184,
       "judge": -31.57894736842105,
       "standard": -9.859154929577462
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -10.75981978481908,
       "hard": -29.341375883900135,
       "judge": -28.232663703069093,
       "standard": -55.86396111215961
      }
     }
    }
   },
   "jevbench_score": 3.5539240822933435e-06,
   "key": "clm-8b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (code and CLM-v0.1-8B head); Qwen3-8B encoder Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6910994764397905,
    "decisive_rate": 0.5093333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 40.041900621338065,
     "cost": 50.501662240594015,
     "intelligence": 0.13075619159747265,
     "speed": 92.63548383949004
    },
    "board_release": "v1.6.43",
    "capability": 20.086328406467768,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 133,
    "capability_rank": 163,
    "composite_ci95": {
     "A": [
      0,
      0.010708585645302497
     ],
     "B": [
      0,
      0.007040756824095157
     ],
     "C": [
      0,
      0.007436517809237845
     ]
    },
    "jevbench_score": 3.5510969933114796e-06,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 193,
    "rank": 237,
    "ranks": {
     "A": 237,
     "B": 237,
     "C": 237,
     "capability": 235
    },
    "scores": {
     "A": 3.5510969933114796e-06,
     "B": 2.2274698008760027e-06,
     "C": 2.466039578688528e-06
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19238147847354412,
     "p50_s_raw": 0.02119073923677206,
     "p95_s_adjusted": 0.2833272576332092,
     "p95_s_raw": 0.0666636288166046
    },
    "views": {
     "Balanced 33:33:33": 2.67198512793619e-06,
     "Emphasis on Accuracy 60:20:20": 1.488390058441499e-06,
     "Emphasis on Cost 20:20:60": 4.430457758214625e-06,
     "Emphasis on Speed 20:60:20": 4.4408219452037955e-06,
     "Intelligence only": 8.942249433748766e-07
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 278,
   "ranked": true,
   "ranks": {
    "A": 278,
    "B": 278,
    "C": 278,
    "capability": 276
   },
   "repo": "https://github.com/Contrastive-LM/CLM",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.5539240822933435e-06,
    "B": 2.2283593211082235e-06,
    "C": 2.4680028349259335e-06
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22330221589654684,
    "p50_s_raw": 0.03665110794827342,
    "p95_s_adjusted": 0.30123862307518723,
    "p95_s_raw": 0.0756193115375936
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      35.37121914730655,
      44.441323364577684
     ],
     "I": [
      0,
      1.9510432807841849
     ],
     "capability": [
      17.69959643896708,
      22.526253333840447
     ],
     "composite": null,
     "seed": 22
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 40.03295405037662,
     "capability": 20.081855120987044,
     "intelligence": 0.13075619159747265
    },
    "per_type": {
     "choice": {
      "accuracy": 0.20533333333333334,
      "calibration": 20.10294495038398,
      "cc": -1.6265168667052297,
      "cc_by_type": {
       "choice": -1.6265168667052297
      },
      "cc_tiered": 0.39226857479241795,
      "cc_tiered_open": 0.7845371495848359,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0
     },
     "noul": {
      "accuracy": 0.352,
      "calibration": 63.079088192783715,
      "cc": -29.600000000000005,
      "cc_by_type": {
       "noul": -29.600000000000005
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.26666666666666666,
      "calibration": 36.91682900796216,
      "cc": -29.29970040054666,
      "cc_by_type": {
       "score": -29.29970040054666
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 38.47490938800186,
     "capability": 19.538705486965597,
     "intelligence": 0.6025015859293292
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0013333333333332975,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.674121210442845e-06,
    "Emphasis on Accuracy 60:20:20": 1.4887875277789586e-06,
    "Emphasis on Cost 20:20:60": 4.441169673531417e-06,
    "Emphasis on Speed 20:60:20": 4.44423793018498e-06,
    "Intelligence only": 8.942249433748766e-07
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "iamupd / corners-ai (iamupd@gmail.com)",
   "axes": {
    "calibration": 81.18327994577871,
    "cost": 79.71332942596933,
    "intelligence": 32.65682750200105,
    "speed": 91.2364277414064
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12863454680642747,
      "ece_score": 74.2730906387145,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 74.2730906387145,
      "score_before_time_limit": 74.2730906387145,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.444743935309973,
      "brier": 0.11602338129808766,
      "ece": 0.08236478989729613,
      "n": 371,
      "score": 83.52704202054078,
      "score_before_time_limit": 83.52704202054078,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.997289972899729,
      "interval90_width": 0.6962511291779581,
      "n": 369,
      "nrps": 0.09343875797512342,
      "score": 85.74970717808084,
      "score_before_time_limit": 85.74970717808084,
      "time_limit_failed_share": 0,
      "top_ece": 0.09578354923162988
     }
    },
    "score": 81.18327994577871
   },
   "capability": 56.92005372388988,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 713,202 input + 0 output tokens over 1,477 decisions = 482.872037915 tokens/decision. GPU-time cost USD 0.00474484852436442/1,000; old list-price cost USD 0.0144861611374408/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00474484852436442/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.014264039999999997
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004744848524364423
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "CoCo-Decision-4B (corners-ai, LoRA on Qwen3.5-4B, option-label logits of one pass, served by oh-my-jev)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,477 of 1,500 answered. LoRA adapter corners-ai/CoCo-Decision-4B-Ko tag v1.0.0 = commit b580ede993493690212a07ed0e6cb972868cc808 (the submission's 'Version 1.0.0', the card's own pin; Apache-2.0) on Qwen/Qwen3.5-4B @ 851bf6e8 as its own adapter_config.json names it. Toolkit github.com/iamupd/oh-my-jev @ f0b8c786 (Apache-2.0) installed with its README's `uv sync` plus the semif extra the README names for --adapter (torch 2.9.1+cu128, transformers 5.17.0, peft 0.21.0). Served by the author's own `omj serve --backend semif --adapter <adapter dir>`: the README documents that --adapter runs locally on semif and loads the adapter's base, in 4-bit only when it would not fit; on the 32 GB card omj itself chose bf16 ('Using Qwen/Qwen3.5-4B (the adapter's base model, bf16)'), the card's headline configuration. NO config.toml was authored and no setting of ours was chosen, which removes run 58's blocker. UNCHANGED typesafe adapter; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. THE 23 LONG ITEMS ARE REFUSED with the documented 422 context_length_exceeded at omj serve's default 4,096-token limit (README 'Known limitations'); they are the system's own disclosed refusal, counted wrong, and no other item failed. One RTX 5090 32 GB (Lium, France). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 619F3346 (bh-submit 0195e1e7)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 32.635837624396466,
    "I_sealed": 32.67781737960564,
    "base": 32.65682750200105,
    "excess": -2.6153673976474203,
    "gap": -0.04197975520917652,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 56.102191211482165,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 54.70273670965714,
       "judge": 68.35860250494396,
       "standard": 32.18920050230222
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 4.347826086956519,
       "judge": -30.000000000000004,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 41.80532166170725,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.92275346241979,
       "hard": 29.940973271546778,
       "judge": 47.18651389666748,
       "standard": 53.90351418923157
      }
     },
     "sealed|choice": {
      "cc": 60.224217892665166,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 76.05498484960461,
       "hard": 58.76696832579185,
       "judge": 56.238361266294234,
       "standard": 61.20211848749847
      }
     },
     "sealed|noul": {
      "cc": 1.5482925144119708,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48,
       "hard": -20.38834951456311,
       "judge": 7.8947368421052655,
       "standard": 12.676056338028175
      }
     },
     "sealed|score": {
      "cc": 36.26094173173978,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 57.8083031267016,
       "hard": 30.22640846893988,
       "judge": 36.4425387268291,
       "standard": 37.283932067224676
      }
     }
    }
   },
   "jevbench_score": 25.680720840435747,
   "key": "coco-decision-4b",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "corners-ai/CoCo-Decision-4B-Ko@b580ede993493690212a07ed0e6cb972868cc808",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9611650485436893,
    "decisive_rate": 0.555256064690027,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.18327994577871,
     "cost": 65.17140064644872,
     "intelligence": 32.65682750200105,
     "speed": 91.3615355688411
    },
    "board_release": "v1.6.43",
    "capability": 56.92005372388988,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 73,
    "capability_rank": 99,
    "composite_ci95": {
     "A": [
      19.632668233405447,
      33.50810171156304
     ],
     "B": [
      16.766044087628412,
      29.654016739790904
     ],
     "C": [
      13.633797384309343,
      23.269515077474335
     ]
    },
    "jevbench_score": 24.64791535308594,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 80,
    "rank": 113,
    "ranks": {
     "A": 113,
     "B": 122,
     "C": 128,
     "capability": 153
    },
    "scores": {
     "A": 24.64791535308594,
     "B": 21.361319353519175,
     "C": 17.116607884087454
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23897131998674012,
     "p50_s_raw": 0.044485659993370064,
     "p95_s_adjusted": 0.30584448990703095,
     "p95_s_raw": 0.07792224495351546
    },
    "views": {
     "Balanced 33:33:33": 22.48701917025063,
     "Emphasis on Accuracy 60:20:20": 18.052162943961683,
     "Emphasis on Cost 20:20:60": 24.348744693137398,
     "Emphasis on Speed 20:60:20": 27.066970501623096,
     "Intelligence only": 13.930989601396824
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 176,
   "ranked": true,
   "ranks": {
    "A": 176,
    "B": 177,
    "C": 178,
    "capability": 185
   },
   "repo": "https://huggingface.co/corners-ai/CoCo-Decision-4B-Ko",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 25.680720840435747,
    "B": 21.974035049217584,
    "C": 17.83383391696927
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.240418037987547,
    "p50_s_raw": 0.0452090189937735,
    "p95_s_adjusted": 0.3128889149724273,
    "p95_s_raw": 0.08144445748621365
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.60430858133772,
      83.74029534027535
     ],
     "I": [
      29.768152266734436,
      37.070992355300035
     ],
     "capability": [
      54.51166132245717,
      59.43802140351146
     ],
     "composite": null,
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.18327994577871,
     "capability": 56.92005372388988,
     "intelligence": 32.65682750200105
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6946666666666667,
      "calibration": 74.2730906387145,
      "cc": 60.9522275797391,
      "cc_by_type": {
       "choice": 60.9522275797391
      },
      "cc_tiered": 58.163204552073665,
      "cc_tiered_open": 56.102191211482165,
      "cc_tiered_sealed": 60.224217892665166,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.9522275797391
     },
     "noul": {
      "accuracy": 0.528,
      "calibration": 83.52704202054078,
      "cc": 5.600000000000005,
      "cc_by_type": {
       "noul": 5.600000000000005
      },
      "cc_tiered": 0.7741462572059854,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 1.5482925144119708,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.600000000000005
     },
     "score": {
      "accuracy": 0.5786666666666667,
      "calibration": 85.74970717808084,
      "cc": 37.72075365036465,
      "cc_by_type": {
       "score": 37.72075365036465
      },
      "cc_tiered": 39.033131696723515,
      "cc_tiered_open": 41.80532166170725,
      "cc_tiered_sealed": 36.26094173173978,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 37.72075365036465
     }
    },
    "raw_on_A": {
     "calibration": 81.23470206450914,
     "capability": 59.56576106652099,
     "intelligence": 37.89682006853285
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 23.643710255039277,
    "Emphasis on Accuracy 60:20:20": 18.487811079235545,
    "Emphasis on Cost 20:20:60": 26.925287092592985,
    "Emphasis on Speed 20:60:20": 28.047234366143474,
    "Intelligence only": 13.930989601396824
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "adimyth",
   "axes": {
    "calibration": 83.26568672143415,
    "cost": 72.14079174579808,
    "intelligence": 28.917433770533798,
    "speed": 83.0508382606611
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08397997501761144,
      "ece_score": 83.2040049964777,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 83.2040049964777,
      "score_before_time_limit": 83.2040049964777,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.7466666666666666,
      "brier": 0.1702847698640915,
      "ece": 0.10141246114807827,
      "n": 375,
      "score": 79.71750777038434,
      "score_before_time_limit": 79.71750777038434,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9413333333333334,
      "interval90_width": 0.5966666666666666,
      "n": 375,
      "nrps": 0.12226058004692472,
      "score": 86.87554739744039,
      "score_before_time_limit": 86.87554739744039,
      "time_limit_failed_share": 0,
      "top_ece": 0.07011423600213387
     }
    },
    "score": 83.26568672143415
   },
   "capability": 56.09156024598397,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,282,261 input + 0 output tokens over 1,485 decisions = 863.475420875 tokens/decision. GPU-time cost USD 0.00848477392532203/1,000; old list-price cost USD 0.0259042626262626/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00848477392532203/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.04933552
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.008484773925322034
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Compass 0.2.0 (adimyth, Qwen3.5-4B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's compass.server: per-candidate verification branches, readout fusion 0.5, debias 0.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 31.733302625698396,
    "I_sealed": 26.1015649153692,
    "base": 28.917433770533798,
    "excess": -4.2077890600265135,
    "gap": 5.6317377103291975,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 62.41376749591724,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 64.1396665618119,
       "judge": 60.44825313117996,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -4.347826086956519,
       "judge": -80,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 32.78614038117795,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.20061237036324,
       "hard": 36.58381707085273,
       "judge": 21.473132873563948,
       "standard": 32.953062268656744
      }
     },
     "sealed|choice": {
      "cc": 58.085681342882154,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 55.24459350063606,
       "judge": 54.162585700293846,
       "standard": 61.08415946002954
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -27.586206896551722,
       "hard": -47.27272727272728,
       "judge": -84.61538461538461,
       "standard": -55.223880597014926
      }
     },
     "sealed|score": {
      "cc": 20.21901340322545,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 39.1090569064439,
       "hard": 15.992779208880192,
       "judge": 13.206706256149614,
       "standard": 29.74492076092049
      }
     }
    }
   },
   "jevbench_score": 18.45610384623482,
   "key": "compass-0.2.0",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "adimyth/compass-lora-v2@bfa8af07916df491fde171d1da0d6d761304f2f3",
   "noul_decisive": {
    "acc_among_decisive": 0.968421052631579,
    "decisive": 95,
    "decisive_rate": 0.25333333333333335,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.26568672143415,
     "cost": 57.598862966277466,
     "intelligence": 28.917433770533798,
     "speed": 83.28951948101809
    },
    "board_release": "v1.6.43",
    "capability": 56.09156024598397,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 79,
    "capability_rank": 105,
    "composite_ci95": {
     "A": [
      12.450188556707262,
      22.048694801053795
     ],
     "B": [
      10.44380870456473,
      19.266813296946893
     ],
     "C": [
      8.645964275491151,
      15.311593611842914
     ]
    },
    "jevbench_score": 17.614153784977713,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 108,
    "rank": 144,
    "ranks": {
     "A": 144,
     "B": 148,
     "C": 147,
     "capability": 160
    },
    "scores": {
     "A": 17.614153784977713,
     "B": 15.129697598253454,
     "C": 12.23205123956786
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.6588797099888325,
     "p50_s_raw": 0.25443985499441624,
     "p95_s_adjusted": 0.7116097962949425,
     "p95_s_raw": 0.2808048981474712
    },
    "views": {
     "Balanced 33:33:33": 15.691595815525753,
     "Emphasis on Accuracy 60:20:20": 12.564181595428897,
     "Emphasis on Cost 20:20:60": 16.94946722349077,
     "Emphasis on Speed 20:60:20": 19.013249270056317,
     "Intelligence only": 9.672511174017334
    }
   },
   "rank": 191,
   "ranked": true,
   "ranks": {
    "A": 191,
    "B": 192,
    "C": 192,
    "capability": 193
   },
   "repo": "https://huggingface.co/adimyth/compass-lora-v2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 18.45610384623482,
    "B": 15.61932651938723,
    "C": 12.816738782107516
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.6632696912623942,
    "p50_s_raw": 0.2566348456311971,
    "p95_s_adjusted": 0.7468373369425535,
    "p95_s_raw": 0.29841866847127674
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6363636363636364,
       "cc": 25.16281616679272,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 0,
        "score": 17.15511516704482
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 25.16281616679272
      },
      "calibration_gen": {
       "accuracy": 0.36633663366336633,
       "cc": -10.700517157662324,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -60,
        "score": 1.0691802343301027
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.44554455445544555,
       "cc": -3.1609446697062276,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -70.37037037037037,
        "score": -6.592138435496286
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 77.56082309299116,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 72.68246927897344
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 77.56082309299116
      },
      "jaggedness": {
       "accuracy": 0.4166666666666667,
       "cc": -7.521417581149862,
       "cc_by_type": {
        "choice": 43.69586140519731,
        "noul": -72.97297297297297,
        "score": 6.712858824326073
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.5591054313099042,
       "cc": 18.55523558036446,
       "cc_by_type": {
        "choice": 68.3014880690323,
        "noul": -54.639175257731964,
        "score": 42.00339392979304
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 18.55523558036446
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": -25.04602397371077,
       "cc_by_type": {
        "choice": 22.58064516129032,
        "noul": -100,
        "score": 2.2812829175773763
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -33.53428485635812,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -100,
        "score": -14.004916424744462
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5745454545454546,
       "cc": 20.339012444839206,
       "cc_by_type": {
        "choice": 67.72089990218454,
        "noul": -41.666666666666664,
        "score": 34.96280409899973
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 20.339012444839206
      },
      "negation_pair_gen": {
       "accuracy": 0.37209302325581395,
       "cc": -25.591151403349343,
       "cc_by_type": {
        "choice": 53.333333333333336,
        "noul": -100,
        "score": -30.106787543381365
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.75,
       "cc": 68.04260985352863,
       "cc_by_type": {
        "choice": 68.04260985352863
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 68.04260985352863
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 79.40020031093738,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100,
        "score": 62.22462495683616
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 79.40020031093738
      },
      "usecase": {
       "accuracy": 0.4772727272727273,
       "cc": 9.25295253481841,
       "cc_by_type": {
        "choice": 55.88235294117649,
        "noul": -54.16666666666667,
        "score": 26.043171329945412
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 9.25295253481841
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -22.487984477627545,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": -50,
        "score": -41.27347724240644
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": -10.526315789473685,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.46153846153846156,
       "cc": -7.328512183344543,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -100,
        "score": 19.48911782784656
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 34.097952211043115,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": -25,
        "score": 35.25845840304084
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 34.097952211043115
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 36.12028010447652,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 91.97350325570144
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 36.12028010447652
      },
      "en": {
       "accuracy": 0.5125977410947002,
       "cc": 9.510128503054053,
       "cc_by_type": {
        "choice": 57.60364164459792,
        "noul": -51.798561151079134,
        "score": 22.72530501564337
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 9.510128503054053
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 44.557456303481054,
       "cc_by_type": {
        "choice": 100,
        "noul": 25,
        "score": 8.672368910443161
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 44.557456303481054
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -22.77211870429048,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": 30.04429962483347
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 9.884178877858309,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": -100,
        "score": 50.82900722181021
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 9.884178877858309
      },
      "hi": {
       "accuracy": 0.2857142857142857,
       "cc": -23.991596849230035,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100,
        "score": -21.695460938751566
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.6666666666666666,
       "cc": 21.774974165868226,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": -33.333333333333336,
        "score": 12.68787391355688
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 21.774974165868226
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 16.651900813212105,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": -40,
        "score": 32.846885918587866
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 16.651900813212105
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -1.514851221719643,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": -4.544553665158935
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.9726455172006,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 97.5846032182685
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.9726455172006
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 4.0277366115052375,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 38.05723580854167
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 4.0277366115052375
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5384615384615384,
       "cc": 15.744381156105492,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -60,
        "score": 30.085449858498457
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 15.744381156105492
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 25.49445136170787,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": -19.999999999999996,
        "score": 49.51812132142296
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 25.49445136170787
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -20.332935033810834,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100,
        "score": 41.12885447303558
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 36.968154265366756,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 10.904462796100267
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 36.968154265366756
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 49.387703797391886,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 12.679240424433747
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 49.387703797391886
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -3.515405794221941,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -26.365426422213844
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6893333333333334,
       "cc": 59.872940546126024,
       "cc_by_type": {
        "choice": 59.872940546126024
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 59.872940546126024
      },
      "noul": {
       "accuracy": 0.24533333333333332,
       "cc": -50.933333333333344,
       "cc_by_type": {
        "noul": -50.933333333333344
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.4666666666666667,
       "cc": 24.379746613482347,
       "cc_by_type": {
        "score": 24.379746613482347
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 24.379746613482347
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -27.575892260550187,
       "cc_by_type": {
        "choice": 52,
        "noul": -100,
        "score": -34.727676781650565
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.20833333333333334,
       "cc": -33.860695446041085,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -100,
        "score": -27.04792484744003
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 35.210411379905146,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": 38.15155934296747
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 35.210411379905146
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -18.69170604393453,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": 14.100320464687643
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": 10.857814798771154,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 23.87779222240042
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 10.857814798771154
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -38.8677190566742,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -100,
        "score": -53.71007540901631
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": 1.1070274802771773,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -71.42857142857143,
        "score": 23.769261712540214
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 1.1070274802771773
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -14.364855425973063,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100,
        "score": 30.076165429397882
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 18.20005544196467,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -60,
        "score": 35.43349965922733
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.20005544196467
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": 0.6193288792792195,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": -33.333333333333336,
        "score": 12.279114403719172
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0.6193288792792195
      },
      "lead_generation": {
       "accuracy": 0.56,
       "cc": 21.46736464644896,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": 9.363334249424426
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 21.46736464644896
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -19.884112566167715,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": 4.863791333754919
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 14.995552254048038,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -60,
        "score": 39.53560109995602
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 14.995552254048038
      },
      "model_routing": {
       "accuracy": 0.5576036866359447,
       "cc": 13.046336664137272,
       "cc_by_type": {
        "choice": 77.7922077922078,
        "noul": -66.66666666666667,
        "score": 28.01346886687068
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 13.046336664137272
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": -0.6512459807687018,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100,
        "score": 55.627835954047065
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 51.828105924296004,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 38.86407636156023
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 51.828105924296004
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -17.472858265243993,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -100,
        "score": 6.694725696878856
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": -3.514125054614875,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": -100,
        "score": 18.443132082532188
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 70.03763968705526,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50,
        "score": 70.78907564479567
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 70.03763968705526
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.62672822912583,
      84.7616620699159
     ],
     "I": [
      25.13342857907104,
      31.751819803526313
     ],
     "capability": [
      52.68979192721088,
      57.180429826061115
     ],
     "composite": null,
     "seed": 27
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.26568672143415,
     "capability": 56.09156024598397,
     "intelligence": 28.917433770533798
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6893333333333334,
      "calibration": 83.2040049964777,
      "cc": 59.872940546126024,
      "cc_by_type": {
       "choice": 59.872940546126024
      },
      "cc_tiered": 60.2497244193997,
      "cc_tiered_open": 62.41376749591724,
      "cc_tiered_sealed": 58.085681342882154,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.872940546126024
     },
     "noul": {
      "accuracy": 0.24533333333333332,
      "calibration": 79.71750777038434,
      "cc": -50.933333333333344,
      "cc_by_type": {
       "noul": -50.933333333333344
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4666666666666667,
      "calibration": 86.87554739744039,
      "cc": 24.379746613482347,
      "cc_by_type": {
       "score": 24.379746613482347
      },
      "cc_tiered": 26.502576892201702,
      "cc_tiered_open": 32.78614038117795,
      "cc_tiered_sealed": 20.21901340322545,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 24.379746613482347
     }
    },
    "raw_on_A": {
     "calibration": 79.36406944543388,
     "capability": 54.5556229058179,
     "intelligence": 29.747176366201913
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.590579762033734,
    "Emphasis on Accuracy 60:20:20": 12.899995182592418,
    "Emphasis on Cost 20:20:60": 18.960271999255177,
    "Emphasis on Speed 20:60:20": 19.77682290767988,
    "Intelligence only": 9.672511174017334
   }
  },
  {
   "adapter_id": "images/cygnet/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "blockbrain",
   "axes": {
    "calibration": 86.96753988280044,
    "cost": 66.80610984255793,
    "intelligence": 54.83928122751086,
    "speed": 91.37843430750362
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08631856803856107,
      "ece_score": 82.73628639228778,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 82.73628639228778,
      "score_before_time_limit": 82.73628639228778,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.18867924528301883,
      "brier": 0.1059190859395021,
      "ece": 0.061302788041812155,
      "n": 371,
      "score": 87.73944239163757,
      "score_before_time_limit": 87.73944239163757,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.978319783197832,
      "interval90_width": 0.46589882565492324,
      "n": 369,
      "nrps": 0.07366857377747793,
      "score": 90.42689086447598,
      "score_before_time_limit": 90.42689086447598,
      "time_limit_failed_share": 0,
      "top_ece": 0.05889680446650118
     }
    },
    "score": 86.96753988280044
   },
   "capability": 70.90341055515565,
   "class": "system-one-open",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 920,404 input + 1,624 output tokens over 1,624 decisions = 567.751231527 tokens/decision. GPU-time cost USD 0.0127780035734122/1,000; old list-price cost USD 0.0283375615763547/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0127780035734122/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01277800357341222
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Cygnet (blockbrain, frozen Gemma-4-12B-it)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 58.329951877214064,
    "I_sealed": 51.348610577807655,
    "base": 54.83928122751086,
    "excess": 4.407953656968166,
    "gap": 6.9813412994064095,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 72.62577711735155,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 60.44825313117996,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 57.7536231884058,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 65.21739130434783,
       "judge": 50,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 44.61045532588486,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.78845800105967,
       "hard": 46.33324418399161,
       "judge": 48.54954649888648,
       "standard": 31.66723951258148
      }
     },
     "sealed|choice": {
      "cc": 65.49755292537436,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 84.03665656640308,
       "hard": 56.730769230769226,
       "judge": 66.4804469273743,
       "standard": 72.28722749107035
      }
     },
     "sealed|noul": {
      "cc": 39.63579350399792,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": 12.621359223300965,
       "judge": 44.736842105263165,
       "standard": 71.83098591549295
      }
     },
     "sealed|score": {
      "cc": 48.9124853040507,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.44540243240824,
       "hard": 39.569328601993966,
       "judge": 55.630472156586585,
       "standard": 47.75535986518157
      }
     }
    }
   },
   "jevbench_score": 71.88318248005169,
   "key": "cygnet",
   "last_measured_on": "2026-10-02",
   "licence": "shim MIT; weights Apache-2.0 with Google's Gemma Prohibited Use Policy",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9169435215946844,
    "decisive_rate": 0.8113207547169812,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.96753988280044,
     "cost": 56.42912569481113,
     "intelligence": 54.83928122751086,
     "speed": 91.77283664982562
    },
    "board_release": "v1.6.43",
    "capability": 70.90341055515565,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 23,
    "capability_rank": 40,
    "composite_ci95": {
     "A": [
      64.42215454949937,
      69.93947552798573
     ],
     "B": [
      60.49166104427309,
      67.44884553134193
     ],
     "C": [
      44.73760732604123,
      67.5344744697005
     ]
    },
    "jevbench_score": 68.55012424151383,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 16,
    "rank": 33,
    "ranks": {
     "A": 33,
     "B": 41,
     "C": 44,
     "capability": 70
    },
    "scores": {
     "A": 68.55012424151383,
     "B": 65.28560070195185,
     "C": 57.26499844882472
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.22363614114001393,
     "p50_s_raw": 0.03681807057000697,
     "p95_s_adjusted": 0.29728592303581536,
     "p95_s_raw": 0.07364296151790768
    },
    "views": {
     "Balanced 33:33:33": 64.03016157894142,
     "Emphasis on Accuracy 60:20:20": 60.00734907512389,
     "Emphasis on Cost 20:20:60": 60.756582833038394,
     "Emphasis on Speed 20:60:20": 72.83760421908612,
     "Intelligence only": 54.83928122751087
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 60,
   "ranked": true,
   "ranks": {
    "A": 60,
    "B": 73,
    "C": 86,
    "capability": 90
   },
   "repo": "https://github.com/blockbrain-ai/cygnet-recipe",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.88318248005169,
    "B": 67.67645067535219,
    "C": 60.049348980229354
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23948653005063533,
    "p50_s_raw": 0.04474326502531767,
    "p95_s_adjusted": 0.3040013258811086,
    "p95_s_raw": 0.07700066294055431
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.02033013748918,
      88.42190039096631
     ],
     "I": [
      49.43932563439576,
      59.02934294021137
     ],
     "capability": [
      66.3438265961016,
      72.89464887942813
     ],
     "composite": null,
     "seed": 23
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.96753988280044,
     "capability": 70.90341055515565,
     "intelligence": 54.83928122751086
    },
    "per_type": {
     "choice": {
      "accuracy": 0.748,
      "calibration": 82.73628639228778,
      "cc": 67.7727991815314,
      "cc_by_type": {
       "choice": 67.7727991815314
      },
      "cc_tiered": 69.06166502136296,
      "cc_tiered_open": 72.62577711735155,
      "cc_tiered_sealed": 65.49755292537436,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.7727991815314
     },
     "noul": {
      "accuracy": 0.736,
      "calibration": 87.73944239163757,
      "cc": 47.199999999999996,
      "cc_by_type": {
       "noul": 47.199999999999996
      },
      "cc_tiered": 48.69470834620186,
      "cc_tiered_open": 57.7536231884058,
      "cc_tiered_sealed": 39.63579350399792,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 47.199999999999996
     },
     "score": {
      "accuracy": 0.6133333333333333,
      "calibration": 90.42689086447598,
      "cc": 48.18037785294212,
      "cc_by_type": {
       "score": 48.18037785294212
      },
      "cc_tiered": 46.76147031496778,
      "cc_tiered_open": 44.61045532588486,
      "cc_tiered_sealed": 48.9124853040507,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 48.18037785294212
     }
    },
    "raw_on_A": {
     "calibration": 83.24254074674259,
     "capability": 69.54334852201868,
     "intelligence": 55.844156297294745
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 67.9543318242646,
    "Emphasis on Accuracy 60:20:20": 62.02127013765572,
    "Emphasis on Cost 20:20:60": 67.49033898910176,
    "Emphasis on Speed 20:60:20": 75.71822722378796,
    "Intelligence only": 54.83928122751087
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "JohnP1 (Hugging Face)",
   "axes": {
    "calibration": 82.13235390405904,
    "cost": 84.02860926700725,
    "intelligence": 8.855548462175122,
    "speed": 93.26967906722538
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0801831746031745,
      "ece_score": 83.9633650793651,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 82.84385354497357,
      "score_before_time_limit": 83.9633650793651,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.7146666666666667,
      "brier": 0.22347219184000008,
      "ece": 0.08123893333333333,
      "n": 375,
      "score": 83.30553486222222,
      "score_before_time_limit": 83.75221333333333,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.8986666666666666,
      "interval90_width": 0.7457777777777779,
      "n": 375,
      "nrps": 0.16816522745333315,
      "score": 80.24767330498135,
      "score_before_time_limit": 80.89483196066668,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.10696906666666664
     }
    },
    "score": 82.13235390405904
   },
   "capability": 45.49395118311708,
   "code_revision": "Author server jonpol01/d1a@aa35ec9928157bba999ccdcf1f4929403ebb0789, run unmodified using its native /v1/systemone endpoint and PyTorch BF16. JohnP1/d1a-e2b@42f61942442f06b9f4c572c071b63684f3ac0dea is an unmerged LoRA with adapter dimension 16 plus a fp32 pointer head and learned temperature on google/gemma-4-E2B@d29ff6b45f081a49ee2733a859c9c9c2d95d1a6f. No quantisation, vLLM or SGLang. The checkpoint's learned temperature and server defaults were used; feedback, learning, prefix cache, CUDA graphs, fused kernels, date facts and chat logging were disabled. No request use_case was injected. Dependencies were pinned and installed offline; author source and weights ran on isolated rented hardware with networking disabled.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 664,666 input + 172,216 output tokens over 1,485 decisions = 563.556902357 tokens/decision. GPU-time cost USD 0.00340705928865373/1,000; old list-price cost USD 0.0205487676767677/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00340705928865373/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.033570026666666676
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0034070592886537333
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "First measurement of JohnP1/d1a-e2b, a Jev-compatible model. The native server returns probabilities for choice, noul and score using a pointer head, without text generation. Adapter and head hashes differ from the other measured Gemma 4 E2B/E4B siblings checked; byte distinctness does not establish independent training. The pinned card and training metadata describe general decisions, developer-tool tasks, long documents, Japanese JGLUE, routing and pull-request labels, with calibration fitted on decision-v7 calibration data and PR-label development data. These are author statements. Training examples and overlap with JevBench were not independently checked. Apache-2.0 is declared on the pinned cards and metadata; no standalone LICENSE file is present in the pinned model trees. The author's server truncates state to its first 65,536 tokens (including BOS and the state delimiter; a question row may use up to 73,728 tokens). Exactly 15 items were answered using a truncated prompt. All 1,500 requests returned HTTP 200 with probability answers; no refusals or errors occurred. Exactly those 15 truncated items exceeded 10 s (maximum 16.2 s); the v1.7 share view counts them as wrong answers with zero calibration. Frozen/off views have no time limit. Server-native input usage after truncation matched the independent tokenizer count on every request; native nonzero output usage counts the serialized probability answer and is priced as reported. Cost is an ESTIMATE, not charged usage or a host quote. The common cost basis excludes the same 15 long items for every row. Self-hosted latency uses the common assumption 2 times measured p50 plus 0.15 s. A 50-request repeat check had identical probabilities and zero argmax flips.",
   "display": "D1A on Gemma 4 E2B (JohnP1 d1a-e2b)",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-26",
    "release": "v1.7.7",
    "results_sha256": "ceb73e57a71a038ff1a864ffffa051286b6d07ab0055e61b1f414dd9c831fa05"
   },
   "endpoint_condition": "Measured 26 Oct 2026 on one rented NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB in Finland over loopback. The two models ran sequentially in fresh isolated containers with source, weights and inputs read-only, no credentials and networking disabled. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 10.46924645157152,
    "I_sealed": 7.241850472778723,
    "base": 8.855548462175122,
    "excess": -6.612130791562914,
    "gap": 3.2273959787927966,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 21.689032398143198,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 33.837837837837846,
       "hard": 20.72978924189997,
       "judge": 16.94133157547792,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": -47.82608695652174,
       "judge": -60,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 9.718706956571364,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.125757575757575,
       "hard": -5.153981481481473,
       "judge": 36.95661016949153,
       "standard": -20.096296296296323
      }
     },
     "sealed|choice": {
      "cc": 21.72555141833617,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 39.00889453621347,
       "hard": 17.774950850005773,
       "judge": 17.140058765915786,
       "standard": 27.8633199746889
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -31.034482758620683,
       "hard": -72.72727272727273,
       "judge": -60,
       "standard": -70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 24.006666666666643,
       "hard": -17.597126760563377,
       "judge": -7.115312499999993,
       "standard": 10.83222727272728
      }
     }
    }
   },
   "jevbench_score": 0.8493910917005856,
   "key": "d1a-gemma4-e2b",
   "last_measured_on": "2026-10-26",
   "licence": "Apache-2.0 declared on the pinned Hugging Face card and metadata; no standalone LICENSE file in the pinned tree. The author GitHub code declares Apache-2.0. Base google/gemma-4-E2B and dataset obligations also apply.",
   "listing": "ranked",
   "measured_in": "v1.7.7",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "JohnP1/d1a-e2b@42f61942442f06b9f4c572c071b63684f3ac0dea + google/gemma-4-E2B@d29ff6b45f081a49ee2733a859c9c9c2d95d1a6f",
   "new_in": "v1.7.7",
   "noul_decisive": {
    "acc_among_decisive": 0.7830188679245284,
    "decisive": 106,
    "decisive_rate": 0.28418230563002683,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 240,
   "ranked": true,
   "ranks": {
    "A": 240,
    "B": 240,
    "C": 240,
    "capability": 226
   },
   "repo": "https://huggingface.co/JohnP1/d1a-e2b",
   "scores": {
    "A": 0.8493910917005856,
    "B": 0.6017439325812588,
    "C": 0.5898549247920732
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21477747400003863,
    "p50_s_raw": 0.03238873700001932,
    "p95_s_adjusted": 0.2193023891001303,
    "p95_s_raw": 0.034651194550065156
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.18181818181818182,
       "cc": -52.72444444444444,
       "cc_by_type": {
        "choice": -8.333333333333334,
        "noul": -100,
        "score": -49.839999999999996
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.25742574257425743,
       "cc": -29.98821071933544,
       "cc_by_type": {
        "choice": -1.802757158006356,
        "noul": -76,
        "score": -12.161874999999966
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.24752475247524752,
       "cc": -35.39739848979686,
       "cc_by_type": {
        "choice": 8.94308943089431,
        "noul": -92.5925925925926,
        "score": -22.542692307692302
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.3541666666666667,
       "cc": -1.726904761904752,
       "cc_by_type": {
        "choice": -14.285714285714285,
        "noul": -19.999999999999996,
        "score": 29.105000000000025
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.2722222222222222,
       "cc": -26.59467568740681,
       "cc_by_type": {
        "choice": 19.15303176130896,
        "noul": -100,
        "score": 1.062941176470611
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.5015974440894568,
       "cc": 22.43474479274934,
       "cc_by_type": {
        "choice": 38.980364532887215,
        "noul": 7.216494845360821,
        "score": 21.107374999999973
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 22.43474479274934
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -15.135639011618395,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -75,
        "score": -20.92238095238097
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.33090909090909093,
       "cc": -13.320449280960311,
       "cc_by_type": {
        "choice": 20.769481578089344,
        "noul": -61.111111111111114,
        "score": 0.3802816901408379
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.20930232558139536,
       "cc": -41.90744444444444,
       "cc_by_type": {
        "choice": 6.666666666666665,
        "noul": -100,
        "score": -32.38899999999998
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.3958333333333333,
       "cc": 22.769640479360852,
       "cc_by_type": {
        "choice": 22.769640479360852
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 22.769640479360852
      },
      "perturb_pair": {
       "accuracy": 0.5,
       "cc": 11.498081543081538,
       "cc_by_type": {
        "choice": 36.93693693693693,
        "noul": 0,
        "score": -2.442692307692318
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 11.498081543081538
      },
      "usecase": {
       "accuracy": 0.32727272727272727,
       "cc": -21.676871633986924,
       "cc_by_type": {
        "choice": 27.696078431372577,
        "noul": -95.83333333333334,
        "score": 3.106639999999994
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": -20.84782661782663,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": -50,
        "score": -17.305384615384646
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.38461538461538464,
       "cc": 3.157228366615456,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -33.333333333333336,
        "score": 5.016999999999983
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 3.157228366615456
      },
      "de": {
       "accuracy": 0.53125,
       "cc": 26.326569193279465,
       "cc_by_type": {
        "choice": 36.283185840707965,
        "noul": 25,
        "score": 17.696521739130432
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 26.326569193279465
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 12.300779019242704,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 20.514999999999993
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 12.300779019242704
      },
      "en": {
       "accuracy": 0.3327541268462207,
       "cc": -13.529936059627692,
       "cc_by_type": {
        "choice": 20.69903955284895,
        "noul": -61.15107913669065,
        "score": -0.1377685950413765
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.4838709677419355,
       "cc": 4.949829830790388,
       "cc_by_type": {
        "choice": 41.27243066884176,
        "noul": 25,
        "score": -51.422941176470594
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 4.949829830790388
      },
      "fi": {
       "accuracy": 0,
       "cc": -50.04918032786886,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100,
        "score": -18.999999999999993
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 25.803405572755423,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 19.76315789473685
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 25.803405572755423
      },
      "hi": {
       "accuracy": 0.3333333333333333,
       "cc": -28.992767690875226,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -100,
        "score": -53.45874999999998
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 36.537080008412815,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 33.33333333333333,
        "score": 4.337142857142839
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 36.537080008412815
      },
      "ja": {
       "accuracy": 0.32,
       "cc": -19.48101120818992,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": -40,
        "score": -32.66066666666665
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -39.8970634920635,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": 13.642142857142836
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.14285714285714285,
       "cc": -12.019444444444439,
       "cc_by_type": {
        "choice": -16.666666666666668,
        "noul": -33.333333333333336,
        "score": 13.941666666666686
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.2,
       "cc": -35.808025974025966,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -100,
        "score": -29.501999999999985
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.4358974358974359,
       "cc": 2.9481544613830053,
       "cc_by_type": {
        "choice": 46.67795175624207,
        "noul": -60,
        "score": 22.16651162790695
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 2.9481544613830053
      },
      "pt": {
       "accuracy": 0.6071428571428571,
       "cc": 35.63641568612542,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": 60.00000000000001,
        "score": 10.55096774193547
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 35.63641568612542
      },
      "sv": {
       "accuracy": 0,
       "cc": -41.8681914893617,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": -100,
        "score": 2.055000000000007
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.38461538461538464,
       "cc": -19.316630434782603,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -50,
        "score": -42.732499999999995
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.2857142857142857,
       "cc": -8.974731182795693,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100,
        "score": 69.85000000000001
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.1,
       "cc": -53.44732580037665,
       "cc_by_type": {
        "choice": -27.11864406779661,
        "noul": -33.333333333333336,
        "score": -99.89000000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.4,
       "cc": 22.50138732084427,
       "cc_by_type": {
        "choice": 22.50138732084427
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 22.50138732084427
      },
      "noul": {
       "accuracy": 0.22133333333333333,
       "cc": -55.733333333333334,
       "cc_by_type": {
        "noul": -55.733333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.36533333333333334,
       "cc": 0.12914259927805372,
       "cc_by_type": {
        "score": 0.12914259927805372
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.12914259927805372
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -39.20509803921569,
       "cc_by_type": {
        "choice": 52,
        "noul": -100,
        "score": -69.61529411764708
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": -21.840848064978488,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100,
        "score": -9.621923076923045
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.21428571428571427,
       "cc": -25.27037940379404,
       "cc_by_type": {
        "choice": -13.821138211382115,
        "noul": -100,
        "score": 38.01
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -29.620146198830415,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -18.685000000000016
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": -3.9305766908212534,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -33.333333333333336,
        "score": 8.498125000000012
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -64.4967714884696,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100,
        "score": -105.43999999999998
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": -5.204690799396677,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -100,
        "score": 13.79769230769231
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.09090909090909091,
       "cc": -34.6021638135636,
       "cc_by_type": {
        "choice": -14.98257839721254,
        "noul": -100,
        "score": 11.176086956521747
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.13333333333333333,
       "cc": -15.763888888888886,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -100,
        "score": 36.04166666666667
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -35.46449581467783,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": -100,
        "score": -16.457727272727276
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.28,
       "cc": -22.092102811056304,
       "cc_by_type": {
        "choice": 18.604651162790695,
        "noul": -77.77777777777779,
        "score": -7.103181818181814
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -28.886209677419355,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -22.1425
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.4074074074074074,
       "cc": -22.80716659557832,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -100,
        "score": 0.6763888888888459
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.5668202764976958,
       "cc": 31.578109217171715,
       "cc_by_type": {
        "choice": 50.909090909090914,
        "noul": 24.242424242424242,
        "score": 19.58281249999998
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 31.578109217171715
      },
      "moderation_trust_safety": {
       "accuracy": 0.3888888888888889,
       "cc": -10.843217793835384,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -100,
        "score": 36.56823529411765
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.3888888888888889,
       "cc": -20.31771230626383,
       "cc_by_type": {
        "choice": 33.147632311977716,
        "noul": -100,
        "score": 5.89923076923079
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -21.668129044721336,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -100,
        "score": -20.66941176470587
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.30434782608695654,
       "cc": -16.264729468599043,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": -66.66666666666667,
        "score": 4.828999999999972
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.375,
       "cc": 2.988686395704564,
       "cc_by_type": {
        "choice": 46.619217081850536,
        "noul": -50,
        "score": 12.346842105263157
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 2.988686395704564
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.29849913244823,
      82.83725762999461
     ],
     "I": [
      5.945409963925079,
      11.615192050362872
     ],
     "capability": [
      41.336463076732606,
      46.540345931586735
     ],
     "composite": null,
     "seed": 28
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.13235390405904,
     "capability": 45.49395118311708,
     "intelligence": 8.855548462175122
    },
    "per_type": {
     "choice": {
      "accuracy": 0.4,
      "calibration": 82.84385354497357,
      "cc": 22.50138732084427,
      "cc_by_type": {
       "choice": 22.50138732084427
      },
      "cc_tiered": 21.707291908239682,
      "cc_tiered_open": 21.689032398143198,
      "cc_tiered_sealed": 21.72555141833617,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 22.50138732084427
     },
     "noul": {
      "accuracy": 0.22133333333333333,
      "calibration": 83.30553486222222,
      "cc": -55.733333333333334,
      "cc_by_type": {
       "noul": -55.733333333333334
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.36533333333333334,
      "calibration": 80.24767330498135,
      "cc": 0.12914259927805372,
      "cc_by_type": {
       "score": 0.12914259927805372
      },
      "cc_tiered": 4.859353478285682,
      "cc_tiered_open": 9.718706956571364,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.12914259927805372
     }
    },
    "raw_on_A": {
     "calibration": 76.01195509676066,
     "capability": 41.82142696618125,
     "intelligence": 7.630898835601833
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.6942660309038701,
    "Emphasis on Accuracy 60:20:20": 0.43399150363832995,
    "Emphasis on Cost 20:20:60": 0.9842745960251357,
    "Emphasis on Speed 20:60:20": 0.9990599418295141,
    "Intelligence only": 0.2777834603240712
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "JohnP1 (Hugging Face)",
   "axes": {
    "calibration": 82.30108318464288,
    "cost": 81.21501802704326,
    "intelligence": 19.912071787053208,
    "speed": 92.42822411239143
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06711619047619045,
      "ece_score": 86.57676190476191,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 85.42240507936509,
      "score_before_time_limit": 86.57676190476191,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.5466666666666666,
      "brier": 0.19734969744000008,
      "ece": 0.09221919999999999,
      "n": 375,
      "score": 81.12119381333333,
      "score_before_time_limit": 81.55616,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9413333333333334,
      "interval90_width": 0.6671111111111107,
      "n": 375,
      "nrps": 0.14081055279777782,
      "score": 80.3596506612302,
      "score_before_time_limit": 81.0077123601111,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.11951760000000003
     }
    },
    "score": 82.30108318464288
   },
   "capability": 51.10657748584804,
   "code_revision": "Author server jonpol01/d1a@aa35ec9928157bba999ccdcf1f4929403ebb0789, run unmodified using its native /v1/systemone endpoint and PyTorch BF16. JohnP1/d1a-e4b@93ee47b2c5ffa96fe7213da6a23c8f53f25a9778 is an unmerged LoRA with adapter dimension 16 plus a fp32 pointer head and learned temperature on google/gemma-4-E4B@411aa17b749aa952df1359d2dcea73917a544d9a. No quantisation, vLLM or SGLang. The checkpoint's learned temperature and server defaults were used; feedback, learning, prefix cache, CUDA graphs, fused kernels, date facts and chat logging were disabled. No request use_case was injected. Dependencies were pinned and installed offline; author source and weights ran on isolated rented hardware with networking disabled.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 4B active parameters; throughput 80585.75 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 664,666 input + 172,302 output tokens over 1,485 decisions = 563.614814815 tokens/decision. GPU-time cost USD 0.00422830265429264/1,000; old list-price cost USD 0.0205545589225589/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00422830265429264/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.033576026666666675
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004228302654292637
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "First measurement of JohnP1/d1a-e4b, a Jev-compatible model. The native server returns probabilities for choice, noul and score using a pointer head, without text generation. Adapter and head hashes differ from the other measured Gemma 4 E2B/E4B siblings checked; byte distinctness does not establish independent training. The pinned card and training metadata describe general decisions, developer-tool tasks, long documents, Japanese JGLUE, routing and pull-request labels, with calibration fitted on decision-v7 calibration data and PR-label development data. These are author statements. Training examples and overlap with JevBench were not independently checked. Apache-2.0 is declared on the pinned cards and metadata; no standalone LICENSE file is present in the pinned model trees. The author's server truncates state to its first 65,536 tokens (including BOS and the state delimiter; a question row may use up to 73,728 tokens). Exactly 15 items were answered using a truncated prompt. All 1,500 requests returned HTTP 200 with probability answers; no refusals or errors occurred. Exactly those 15 truncated items exceeded 10 s (maximum 20.6 s); the v1.7 share view counts them as wrong answers with zero calibration. Frozen/off views have no time limit. Server-native input usage after truncation matched the independent tokenizer count on every request; native nonzero output usage counts the serialized probability answer and is priced as reported. Cost is an ESTIMATE, not charged usage or a host quote. The common cost basis excludes the same 15 long items for every row. Self-hosted latency uses the common assumption 2 times measured p50 plus 0.15 s. A 50-request repeat check had identical probabilities and zero argmax flips.",
   "display": "D1A on Gemma 4 E4B (JohnP1 d1a-e4b)",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-26",
    "release": "v1.7.7",
    "results_sha256": "ceb73e57a71a038ff1a864ffffa051286b6d07ab0055e61b1f414dd9c831fa05"
   },
   "endpoint_condition": "Measured 26 Oct 2026 on one rented NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB in Finland over loopback. The two models ran sequentially in fresh isolated containers with source, weights and inputs read-only, no credentials and networking disabled. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 24.04921685928431,
    "I_sealed": 15.774926714822108,
    "base": 19.912071787053208,
    "excess": -1.5652366258935082,
    "gap": 8.274290144462203,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 44.924155171815144,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 35.82887700534759,
       "judge": 52.537903757415954,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": -30.434782608695656,
       "judge": -40,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 27.223495406037788,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.99909090909092,
       "hard": 21.70185185185187,
       "judge": 33.483559322033905,
       "standard": 16.988888888888887
      }
     },
     "sealed|choice": {
      "cc": 39.82250425976216,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 31.826066843992123,
       "judge": 36.53281096963763,
       "standard": 43.049989453701755
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -6.896551724137934,
       "hard": -36.36363636363637,
       "judge": -41.53846153846153,
       "standard": -7.462686567164178
      }
     },
     "sealed|score": {
      "cc": 7.502275884704162,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 29.868157894736836,
       "hard": 0.7107887323943962,
       "judge": -2.06442708333332,
       "standard": 24.252363636363583
      }
     }
    }
   },
   "jevbench_score": 7.4194035339710895,
   "key": "d1a-gemma4-e4b",
   "last_measured_on": "2026-10-26",
   "licence": "Apache-2.0 declared on the pinned Hugging Face card and metadata; no standalone LICENSE file in the pinned tree. The author GitHub code declares Apache-2.0. Base google/gemma-4-E4B and dataset obligations also apply.",
   "listing": "ranked",
   "measured_in": "v1.7.7",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "JohnP1/d1a-e4b@93ee47b2c5ffa96fe7213da6a23c8f53f25a9778 + google/gemma-4-E4B@411aa17b749aa952df1359d2dcea73917a544d9a",
   "new_in": "v1.7.7",
   "noul_decisive": {
    "acc_among_decisive": 0.8411764705882353,
    "decisive": 170,
    "decisive_rate": 0.45576407506702415,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 223,
   "ranked": true,
   "ranks": {
    "A": 223,
    "B": 225,
    "C": 224,
    "capability": 209
   },
   "repo": "https://huggingface.co/JohnP1/d1a-e4b",
   "scores": {
    "A": 7.4194035339710895,
    "B": 5.842588698775747,
    "C": 5.152363565257702
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23081404799997926,
    "p50_s_raw": 0.04040702399998963,
    "p95_s_adjusted": 0.2476939227000912,
    "p95_s_raw": 0.0488469613500456
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5,
       "cc": -4.098888888888877,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": 0,
        "score": -45.62999999999997
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.39603960396039606,
       "cc": -0.4393244079179901,
       "cc_by_type": {
        "choice": 36.37327677624603,
        "noul": -28.000000000000004,
        "score": -9.691249999999997
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.38613861386138615,
       "cc": -3.16093102170338,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": -18.518518518518523,
        "score": -9.663461538461537
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.5416666666666666,
       "cc": 16.58026455026455,
       "cc_by_type": {
        "choice": 71.42857142857143,
        "noul": -60,
        "score": 38.31222222222222
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 16.58026455026455
      },
      "jaggedness": {
       "accuracy": 0.34444444444444444,
       "cc": -15.22313130942185,
       "cc_by_type": {
        "choice": 30.702598652550538,
        "noul": -72.97297297297297,
        "score": -3.3990196078431234
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.645367412140575,
       "cc": 48.72859111355397,
       "cc_by_type": {
        "choice": 52.452232103548475,
        "noul": 42.26804123711341,
        "score": 51.465500000000006
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 48.72859111355397
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -10.402426771395849,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75,
        "score": 18.019523809523797
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.49818181818181817,
       "cc": 11.33008823801046,
       "cc_by_type": {
        "choice": 47.179654385392894,
        "noul": -33.333333333333336,
        "score": 20.143943661971818
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 11.33008823801046
      },
      "negation_pair_gen": {
       "accuracy": 0.29069767441860467,
       "cc": -28.394111111111112,
       "cc_by_type": {
        "choice": 23.333333333333332,
        "noul": -86.66666666666667,
        "score": -21.848999999999986
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.3958333333333333,
       "cc": 22.769640479360852,
       "cc_by_type": {
        "choice": 22.769640479360852
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 22.769640479360852
      },
      "perturb_pair": {
       "accuracy": 0.75,
       "cc": 55.36142488642489,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 39.99999999999999,
        "score": 59.1173076923077
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 55.36142488642489
      },
      "usecase": {
       "accuracy": 0.42272727272727273,
       "cc": 0.6803762091503435,
       "cc_by_type": {
        "choice": 49.75490196078433,
        "noul": -58.33333333333333,
        "score": 10.619560000000028
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0.6803762091503435
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 2.121611721611719,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": -50,
        "score": 13.507692307692299
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 2.121611721611719
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 13.037337429595489,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": 13.919999999999998
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 13.037337429595489
      },
      "de": {
       "accuracy": 0.625,
       "cc": 43.35441964858279,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 25,
        "score": 52.85086956521741
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 43.35441964858279
      },
      "el": {
       "accuracy": 0.4,
       "cc": 17.341409062693973,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 85.72999999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 17.341409062693973
      },
      "en": {
       "accuracy": 0.4569939183318853,
       "cc": 8.064044173184532,
       "cc_by_type": {
        "choice": 39.59068109981568,
        "noul": -25.899280575539574,
        "score": 10.50073199527749
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 8.064044173184532
      },
      "es": {
       "accuracy": 0.6451612903225806,
       "cc": 41.18226785657168,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 50,
        "score": 2.9105882352941426
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 41.18226785657168
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -21.242440801457196,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": 34.633333333333326
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 29.551300309597522,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 31.006842105263154
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 29.551300309597522
      },
      "hi": {
       "accuracy": 0.42857142857142855,
       "cc": -9.789776536312846,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -75,
        "score": -4.089999999999994
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 41.42171460738101,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 33.33333333333333,
        "score": 4.961428571428572
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 41.42171460738101
      },
      "ja": {
       "accuracy": 0.4,
       "cc": 2.1289887918100825,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": 0,
        "score": -7.8306666666666525
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 2.1289887918100825
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -25.13891534391534,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": 13.472142857142867
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.7857142857142857,
       "cc": 73.67888888888889,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 37.70333333333335
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 73.67888888888889
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -5.84935064935065,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 34.4
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 30.893496901521186,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -19.999999999999996,
        "score": 43.15023255813952
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 30.893496901521186
      },
      "pt": {
       "accuracy": 0.6071428571428571,
       "cc": 42.320394180749076,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": 60.00000000000001,
        "score": 30.602903225806443
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 42.320394180749076
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": -6.960248226950355,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 15.71499999999999
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": -10.077880434782607,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0,
        "score": -65.01625
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 40.0252688172043,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": 100,
        "score": 16.849999999999998
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 40.0252688172043
      },
      "zh": {
       "accuracy": 0.4,
       "cc": -16.931506591337108,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336,
        "score": -41.19000000000004
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5493333333333333,
       "cc": 41.78993092098969,
       "cc_by_type": {
        "choice": 41.78993092098969
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 41.78993092098969
      },
      "noul": {
       "accuracy": 0.38133333333333336,
       "cc": -23.733333333333327,
       "cc_by_type": {
        "noul": -23.733333333333327
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.4186666666666667,
       "cc": 13.589756317689671,
       "cc_by_type": {
        "score": 13.589756317689671
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 13.589756317689671
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.09090909090909091,
       "cc": -49.807647058823534,
       "cc_by_type": {
        "choice": 4,
        "noul": -100,
        "score": -53.422941176470616
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": -16.792386526516964,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100,
        "score": 5.523461538461538
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 21.10989159891599,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": -4.150000000000009
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 21.10989159891599
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -26.09521929824562,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": 26.97749999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 3.6919685990338174,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": -12.112499999999994
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 3.6919685990338174
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -42.79343815513625,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100,
        "score": -40.329999999999956
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": 6.129093586152405,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -71.42857142857143,
        "score": 29.031538461538464
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 6.129093586152405
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -9.975751148815833,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100,
        "score": 43.24347826086957
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.2,
       "cc": -8.34833333333333,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60,
        "score": 18.28833333333334
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 17.832784374797217,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 0,
        "score": 17.738181818181808
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 17.832784374797217
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 5.551225041108765,
       "cc_by_type": {
        "choice": 53.48837209302325,
        "noul": -33.333333333333336,
        "score": -3.501363636363619
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 5.551225041108765
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -52.38723118279571,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100,
        "score": -60.38750000000002
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 0.07877781332194662,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -60,
        "score": -5.214722222222257
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0.07877781332194662
      },
      "model_routing": {
       "accuracy": 0.7142857142857143,
       "cc": 55.566690791847044,
       "cc_by_type": {
        "choice": 69.61038961038962,
        "noul": 51.515151515151516,
        "score": 45.57453124999999
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 55.566690791847044
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": -10.118471642015734,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 4.193529411764696
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 26.91824786324787,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 14.088076923076942
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 26.91824786324787
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -9.704765768376312,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -100,
        "score": 0.4423529411764715
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.34782608695652173,
       "cc": -5.513396135265704,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": -66.66666666666667,
        "score": 37.08299999999999
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.5416666666666666,
       "cc": 19.866650433913964,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": -50,
        "score": 41.62842105263158
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 19.866650433913964
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.78596564192395,
      83.88434134904129
     ],
     "I": [
      15.298710771678644,
      23.06353856577184
     ],
     "capability": [
      46.435191030984484,
      52.665784837478675
     ],
     "composite": null,
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.30108318464288,
     "capability": 51.10657748584804,
     "intelligence": 19.912071787053208
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5493333333333333,
      "calibration": 85.42240507936509,
      "cc": 41.78993092098969,
      "cc_by_type": {
       "choice": 41.78993092098969
      },
      "cc_tiered": 42.37332971578866,
      "cc_tiered_open": 44.924155171815144,
      "cc_tiered_sealed": 39.82250425976216,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 41.78993092098969
     },
     "noul": {
      "accuracy": 0.38133333333333336,
      "calibration": 81.12119381333333,
      "cc": -23.733333333333327,
      "cc_by_type": {
       "noul": -23.733333333333327
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4186666666666667,
      "calibration": 80.3596506612302,
      "cc": 13.589756317689671,
      "cc_by_type": {
       "score": 13.589756317689671
      },
      "cc_tiered": 17.362885645370977,
      "cc_tiered_open": 27.223495406037788,
      "cc_tiered_sealed": 7.502275884704162,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 13.589756317689671
     }
    },
    "raw_on_A": {
     "calibration": 80.35415762313048,
     "capability": 50.69015431454672,
     "intelligence": 21.026151005962966
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 6.486289347627407,
    "Emphasis on Accuracy 60:20:20": 4.562750093414338,
    "Emphasis on Cost 20:20:60": 8.093381055894556,
    "Emphasis on Speed 20:60:20": 8.347926033530758,
    "Intelligence only": 3.157979738758466
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mark Marosi (Mapika)",
   "axes": {
    "calibration": 80.75417276143466,
    "cost": 68.15540194207952,
    "intelligence": 61.94396690132827,
    "speed": 90.45532474538248
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.048897499999999955,
      "ece_score": 90.22050000000002,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 88.65667800000001,
      "score_before_time_limit": 90.22050000000002,
      "time_limit_failed_share": 0.017333333333333333
     },
     "noul": {
      "abstention_rate": 0.013333333333333308,
      "brier": 0.16693694264000003,
      "ece": 0.1691528,
      "n": 375,
      "score": 65.46363264,
      "score_before_time_limit": 66.16944000000001,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.952,
      "interval90_width": 0.33711111111111114,
      "n": 375,
      "nrps": 0.07347944788000003,
      "score": 88.14220764430401,
      "score_before_time_limit": 89.57541427266668,
      "time_limit_failed_share": 0.016,
      "top_ece": 0.06750613333333315
     }
    },
    "score": 80.75417276143466
   },
   "capability": 71.34906983138146,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 831,319 input + 0 output tokens over 1,624 decisions = 511.895935961 tokens/decision. GPU-time cost USD 0.0115209051706081/1,000; old list-price cost USD 0.0255947967980296/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0115209051706081/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01152090517060813
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 23,
    "n_over_limit_answered_ok": 23,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider-12b v2 (Mapika)",
   "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P83 on one evaluator-owned Lium RTX PRO6000. Offline contributor image, serial loopback-only requests, no batching or retry; standard selfhosted x2+0.15 s latency adjustment. Current stage receipt verifies the same model pin used by the prior add-request run. Cost is the carried documented estimate recorded in current registry.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO6000",
   "intelligence": {
    "I_open": 62.171940568331934,
    "I_sealed": 61.7159932343246,
    "base": 61.94396690132827,
    "excess": -2.11744030843091,
    "gap": 0.4559473340073339,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 74.72393862958435,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 69.80182447310474,
       "judge": 76.26895187870797,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 58.42028985507247,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 65.21739130434783,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 53.37159322033899,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 55.93666666666668,
       "hard": 56.19055555555556,
       "judge": 47.0086440677966,
       "standard": 55.995555555555555
      }
     },
     "sealed|choice": {
      "cc": 72.96622719698513,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 66.91176470588235,
       "judge": 77.6536312849162,
       "standard": 74.50424929178472
      }
     },
     "sealed|noul": {
      "cc": 59.149881609534575,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 41.747572815533985,
       "judge": 60.52631578947369,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 53.03187089645412,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 69.177304964539,
       "hard": 46.133137254901946,
       "judge": 55.79760479041915,
       "standard": 54.60802030456852
      }
     }
    }
   },
   "jevbench_score": 73.72671792000273,
   "key": "decider-12b",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Mapika/decider-12b@8ac1efa708b71b86ae33b01d2a8d7a3ddcb48e66",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8333333333333334,
    "decisive_rate": 0.9865229110512129,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.98845142422223,
     "cost": 57.755449421401664,
     "intelligence": 63.01250112161242,
     "speed": 90.47583056481872
    },
    "board_release": "v1.6.43",
    "capability": 72.50047627291733,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 17,
    "capability_rank": 34,
    "composite_ci95": {
     "A": [
      68.54138708333946,
      72.20753634823356
     ],
     "B": [
      66.06197783195658,
      71.11572775091496
     ],
     "C": [
      63.43447024505396,
      72.20753634823356
     ]
    },
    "jevbench_score": 70.87955164338175,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 8,
    "rank": 21,
    "ranks": {
     "A": 21,
     "B": 24,
     "C": 20,
     "capability": 60
    },
    "scores": {
     "A": 70.87955164338175,
     "B": 69.15281897628992,
     "C": 70.87955164338175
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25826345253735783,
     "p50_s_raw": 0.054131726268678904,
     "p95_s_adjusted": 0.3470195889472961,
     "p95_s_raw": 0.09850979447364806
    },
    "views": {
     "Balanced 33:33:33": 67.81664735288307,
     "Emphasis on Accuracy 60:20:20": 65.80968465261019,
     "Emphasis on Cost 20:20:60": 63.39892316578739,
     "Emphasis on Speed 20:60:20": 75.36672286262451,
     "Intelligence only": 63.01250112161242
    }
   },
   "provenance": {
    "carried_metadata_from": "run 46 author metadata + current P83 staged pin/measurement receipt",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 38,
   "ranked": true,
   "ranks": {
    "A": 38,
    "B": 44,
    "C": 33,
    "capability": 84
   },
   "repo": "https://huggingface.co/Mapika/decider-12b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.72671792000273,
    "B": 71.02470680162858,
    "C": 73.72671792000273
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.25852626711130144,
    "p50_s_raw": 0.05426313355565071,
    "p95_s_adjusted": 0.34830751921981573,
    "p95_s_raw": 0.09915375960990787
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.58237996826853,
      82.2031372305055
     ],
     "I": [
      56.7518720519787,
      66.169056099611
     ],
     "capability": [
      66.92392625670921,
      73.80985552870881
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 80.75417276143466,
     "capability": 71.34906983138146,
     "intelligence": 61.94396690132827
    },
    "per_type": {
     "choice": {
      "accuracy": 0.792,
      "calibration": 88.65667800000001,
      "cc": 73.39977075301005,
      "cc_by_type": {
       "choice": 73.39977075301005
      },
      "cc_tiered": 73.84508291328474,
      "cc_tiered_open": 74.72393862958435,
      "cc_tiered_sealed": 72.96622719698513,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.39977075301005
     },
     "noul": {
      "accuracy": 0.8133333333333334,
      "calibration": 65.46363264,
      "cc": 62.66666666666667,
      "cc_by_type": {
       "noul": 62.66666666666667
      },
      "cc_tiered": 58.78508573230352,
      "cc_tiered_open": 58.42028985507247,
      "cc_tiered_sealed": 59.149881609534575,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 62.66666666666667
     },
     "score": {
      "accuracy": 0.6453333333333333,
      "calibration": 88.14220764430401,
      "cc": 53.379105263157925,
      "cc_by_type": {
       "score": 53.379105263157925
      },
      "cc_tiered": 53.20173205839656,
      "cc_tiered_open": 53.37159322033899,
      "cc_tiered_sealed": 53.03187089645412,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.379105263157925
     }
    },
    "raw_on_A": {
     "calibration": 80.60180365288517,
     "capability": null,
     "intelligence": null
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 71.64837046665977,
    "Emphasis on Accuracy 60:20:20": 67.42324622133162,
    "Emphasis on Cost 20:20:60": 70.2090825539625,
    "Emphasis on Speed 20:60:20": 78.14756886678965,
    "Intelligence only": 61.94396690132827
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mark Marosi (Mapika)",
   "axes": {
    "calibration": 82.51700722651287,
    "cost": 68.15540194207952,
    "intelligence": 59.571985034161884,
    "speed": 90.44187377539939
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0649509375000001,
      "ece_score": 87.00981249999998,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 85.50164241666666,
      "score_before_time_limit": 87.00981249999998,
      "time_limit_failed_share": 0.017333333333333333
     },
     "noul": {
      "abstention_rate": 0.040000000000000036,
      "brier": 0.1306117199200001,
      "ece": 0.12608400000000003,
      "n": 375,
      "score": 73.98551253333332,
      "score_before_time_limit": 74.7832,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.9893333333333333,
      "interval90_width": 0.493333333333333,
      "n": 375,
      "nrps": 0.07613894289555556,
      "score": 88.06386672953866,
      "score_before_time_limit": 89.49579952188888,
      "time_limit_failed_share": 0.016,
      "top_ece": 0.06697253333333336
     }
    },
    "score": 82.51700722651287
   },
   "capability": 71.04449613033738,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 831,319 input + 0 output tokens over 1,624 decisions = 511.895935961 tokens/decision. GPU-time cost USD 0.0115209051706081/1,000; old list-price cost USD 0.0255947967980296/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0115209051706081/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01152090517060813
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 23,
    "n_over_limit_answered_ok": 23,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider-12b v1, stock Gemma-4-12B-it (Mapika)",
   "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P83 on one evaluator-owned Lium RTX PRO6000. Offline contributor image, serial loopback-only requests, no batching or retry; standard selfhosted x2+0.15 s latency adjustment. Current stage receipt verifies the same model pin used by the prior add-request run. Cost is the carried documented estimate recorded in current registry.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO6000",
   "intelligence": {
    "I_open": 61.530130814752454,
    "I_sealed": 57.61383925357131,
    "base": 59.571985034161884,
    "excess": 1.3429039187429033,
    "gap": 3.916291561181147,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 69.2197724963132,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 64.1396665618119,
       "judge": 60.44825313117996,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 73.89855072463769,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 70,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 41.4720692233065,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.67363636363637,
       "hard": 39.991944444444435,
       "judge": 43.4459322033898,
       "standard": 35.370740740740736
      }
     },
     "sealed|choice": {
      "cc": 66.2478597142713,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 61.82126696832577,
       "judge": 66.4804469273743,
       "standard": 67.8531838896416
      }
     },
     "sealed|noul": {
      "cc": 57.758775648268774,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 30.097087378640786,
       "judge": 65.78947368421053,
       "standard": 85.91549295774648
      }
     },
     "sealed|score": {
      "cc": 48.83488239817386,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 65.59943262411348,
       "hard": 41.64539215686275,
       "judge": 54.09401197604791,
       "standard": 46.94289340101522
      }
     }
    }
   },
   "jevbench_score": 73.21407386805642,
   "key": "decider-12b-v1",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Mapika/decider-12b@09e9e3387fa42fed740233be2290385d18a6044d",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.865546218487395,
    "decisive_rate": 0.9622641509433962,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.7629373406296,
     "cost": 57.755449421401664,
     "intelligence": 60.62893554403879,
     "speed": 90.47654290454523
    },
    "board_release": "v1.6.43",
    "capability": 72.1959364423342,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 19,
    "capability_rank": 36,
    "composite_ci95": {
     "A": [
      68.08779334144351,
      71.63200792040908
     ],
     "B": [
      65.27927398150561,
      70.07926650409995
     ],
     "C": [
      58.91555277575465,
      71.63200792040908
     ]
    },
    "jevbench_score": 70.42352588714175,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 10,
    "rank": 25,
    "ranks": {
     "A": 25,
     "B": 26,
     "C": 23,
     "capability": 63
    },
    "scores": {
     "A": 70.42352588714175,
     "B": 68.21936151093459,
     "C": 70.42352588714175
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25808657668530943,
     "p50_s_raw": 0.054043288342654705,
     "p95_s_adjusted": 0.3472004604525864,
     "p95_s_raw": 0.0986002302262932
    },
    "views": {
     "Balanced 33:33:33": 66.87360828773896,
     "Emphasis on Accuracy 60:20:20": 64.22748164585495,
     "Emphasis on Cost 20:20:60": 62.90137869162178,
     "Emphasis on Speed 20:60:20": 74.66484030090665,
     "Intelligence only": 60.62893554403879
    }
   },
   "provenance": {
    "carried_metadata_from": "run 46 author metadata + current P83 staged pin/measurement receipt",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 45,
   "ranked": true,
   "ranks": {
    "A": 45,
    "B": 53,
    "C": 43,
    "capability": 89
   },
   "repo": "https://huggingface.co/Mapika/decider-12b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.21407386805642,
    "B": 70.00769684737924,
    "C": 72.17324227522057
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2587531372904778,
    "p50_s_raw": 0.054376568645238876,
    "p95_s_adjusted": 0.3490816324949264,
    "p95_s_raw": 0.09954081624746322
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.40473152183768,
      83.88253007662135
     ],
     "I": [
      54.75241389332252,
      63.49771826954563
     ],
     "capability": [
      66.46482011493742,
      72.9669680184571
     ],
     "composite": null,
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.51700722651287,
     "capability": 71.04449613033738,
     "intelligence": 59.571985034161884
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7466666666666667,
      "calibration": 85.50164241666666,
      "cc": 67.60228489148659,
      "cc_by_type": {
       "choice": 67.60228489148659
      },
      "cc_tiered": 67.73381610529225,
      "cc_tiered_open": 69.2197724963132,
      "cc_tiered_sealed": 66.2478597142713,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.60228489148659
     },
     "noul": {
      "accuracy": 0.824,
      "calibration": 73.98551253333332,
      "cc": 64.8,
      "cc_by_type": {
       "noul": 64.8
      },
      "cc_tiered": 65.82866318645323,
      "cc_tiered_open": 73.89855072463769,
      "cc_tiered_sealed": 57.758775648268774,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 64.8
     },
     "score": {
      "accuracy": 0.64,
      "calibration": 88.06386672953866,
      "cc": 47.483307017543865,
      "cc_by_type": {
       "score": 47.483307017543865
      },
      "cc_tiered": 45.153475810740176,
      "cc_tiered_open": 41.4720692233065,
      "cc_tiered_sealed": 48.83488239817386,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 47.483307017543865
     }
    },
    "raw_on_A": {
     "calibration": 79.88170832181567,
     "capability": null,
     "intelligence": null
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.56235044026305,
    "Emphasis on Accuracy 60:20:20": 65.71302312214345,
    "Emphasis on Cost 20:20:60": 69.57945453574435,
    "Emphasis on Speed 20:60:20": 77.36436181779911,
    "Intelligence only": 59.57198503416188
   }
  },
  {
   "adapter_id": "images/decider-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mapika",
   "axes": {
    "calibration": 68.52438190259257,
    "cost": 85.78804188568367,
    "intelligence": 20.654576369710338,
    "speed": 94.47902882434015
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.18752718750000008,
      "ece_score": 62.49456249999999,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 62.49456249999999,
      "score_before_time_limit": 62.49456249999999,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.31999999999999995,
      "brier": 0.21768861922666663,
      "ece": 0.1706301333333334,
      "n": 375,
      "score": 65.87397333333331,
      "score_before_time_limit": 65.87397333333331,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8746666666666667,
      "interval90_width": 0.5113333333333334,
      "n": 375,
      "nrps": 0.14404540251111114,
      "score": 77.20460987444443,
      "score_before_time_limit": 77.20460987444443,
      "time_limit_failed_share": 0,
      "top_ece": 0.1559312000000001
     }
    },
    "score": 68.52438190259257
   },
   "capability": 44.58947913615145,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 799,606 input + 0 output tokens over 1,624 decisions = 492.368226601 tokens/decision. GPU-time cost USD 0.00297667854455/1,000; old list-price cost USD 0.0147710467980296/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00297667854455/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0029766785445500044
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider-2b (Mapika)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 22.803908585466385,
    "I_sealed": 18.50524415395429,
    "base": 20.654576369710338,
    "excess": 1.7252767890738503,
    "gap": 4.298664431512094,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 37.01745599452806,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 24.50456118276188,
       "judge": 28.80685563612393,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 4.469289164941341,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 4.347826086956519,
       "judge": -19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 26.92498059692975,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.77454545454546,
       "hard": 25.165648148148158,
       "judge": 23.80372881355931,
       "standard": 18.20074074074074
      }
     },
     "sealed|choice": {
      "cc": 39.65284081929382,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 58.7613627965413,
       "hard": 29.24208144796379,
       "judge": 35.7541899441341,
       "standard": 56.76807488606972
      }
     },
     "sealed|noul": {
      "cc": 0.12111092718806482,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -28.155339805825243,
       "judge": 10.526315789473696,
       "standard": 21.126760563380277
      }
     },
     "sealed|score": {
      "cc": 15.741780715380987,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 52.834113475177304,
       "hard": 4.744583333333363,
       "judge": 10.707844311377212,
       "standard": 26.740913705583736
      }
     }
    }
   },
   "jevbench_score": 8.006807575798623,
   "key": "decider-2b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7764705882352941,
    "decisive_rate": 0.68,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 68.52438190259257,
     "cost": 64.91766177740826,
     "intelligence": 20.654576369710338,
     "speed": 94.55945724666857
    },
    "board_release": "v1.6.43",
    "capability": 44.58947913615145,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 101,
    "capability_rank": 129,
    "composite_ci95": {
     "A": [
      4.470553979976864,
      13.010288397584645
     ],
     "B": [
      3.512028417919115,
      10.890754224642697
     ],
     "C": [
      3.1045513749839326,
      9.034922498322668
     ]
    },
    "jevbench_score": 7.6704332123718,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 139,
    "rank": 178,
    "ranks": {
     "A": 178,
     "B": 179,
     "C": 180,
     "capability": 188
    },
    "scores": {
     "A": 7.6704332123718,
     "B": 6.209608793546842,
     "C": 5.326689730813751
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.17949020988307893,
     "p50_s_raw": 0.014745104941539466,
     "p95_s_adjusted": 0.1949905256042257,
     "p95_s_raw": 0.022495262802112847
    },
    "views": {
     "Balanced 33:33:33": 6.881302219979439,
     "Emphasis on Accuracy 60:20:20": 4.98303010129531,
     "Emphasis on Cost 20:20:60": 8.110241448995462,
     "Emphasis on Speed 20:60:20": 8.93000700376026,
     "Intelligence only": 3.5245921294252027
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 222,
   "ranked": true,
   "ranks": {
    "A": 222,
    "B": 223,
    "C": 223,
    "capability": 228
   },
   "repo": "https://huggingface.co/Mapika/decider-2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 8.006807575798623,
    "B": 6.383285716457357,
    "C": 5.560283038749045
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18164706099778413,
    "p50_s_raw": 0.01582353049889207,
    "p95_s_adjusted": 0.19627669751644133,
    "p95_s_raw": 0.02313834875822067
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-2B-Base with a trained decision readout, 1.9B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      62.32738945440369,
      71.2629933305287
     ],
     "I": [
      16.778817874747006,
      25.441954749747556
     ],
     "capability": [
      40.31419112262012,
      47.6366012348925
     ],
     "composite": null,
     "seed": 26
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 68.52438190259257,
     "capability": 44.58947913615145,
     "intelligence": 20.654576369710338
    },
    "per_type": {
     "choice": {
      "accuracy": 0.532,
      "calibration": 62.49456249999999,
      "cc": 40.149484194272596,
      "cc_by_type": {
       "choice": 40.149484194272596
      },
      "cc_tiered": 38.335148406910946,
      "cc_tiered_open": 37.01745599452806,
      "cc_tiered_sealed": 39.65284081929382,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 40.149484194272596
     },
     "noul": {
      "accuracy": 0.528,
      "calibration": 65.87397333333331,
      "cc": 5.600000000000005,
      "cc_by_type": {
       "noul": 5.600000000000005
      },
      "cc_tiered": 2.295200046064703,
      "cc_tiered_open": 4.469289164941341,
      "cc_tiered_sealed": 0.12111092718806482,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.600000000000005
     },
     "score": {
      "accuracy": 0.41333333333333333,
      "calibration": 77.20460987444443,
      "cc": 20.228824561403492,
      "cc_by_type": {
       "score": 20.228824561403492
      },
      "cc_tiered": 21.33338065615537,
      "cc_tiered_open": 26.92498059692975,
      "cc_tiered_sealed": 15.741780715380987,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 20.228824561403492
     }
    },
    "raw_on_A": {
     "calibration": 62.755705376823364,
     "capability": 43.8983266486265,
     "intelligence": 25.040947920429637
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 7.245397624690648,
    "Emphasis on Accuracy 60:20:20": 5.094256509090691,
    "Emphasis on Cost 20:20:60": 9.079768824392326,
    "Emphasis on Speed 20:60:20": 9.291824613361115,
    "Intelligence only": 3.5245921294252027
   }
  },
  {
   "adapter_id": "images/decider-35b-a3b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mapika",
   "axes": {
    "calibration": 83.61830413355554,
    "cost": 73.18143549566516,
    "intelligence": 48.72471958402083,
    "speed": 91.16871958052303
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10659875000000013,
      "ece_score": 78.68024999999997,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 78.68024999999997,
      "score_before_time_limit": 78.68024999999997,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2613333333333333,
      "brier": 0.11783857269333342,
      "ece": 0.07045920000000003,
      "n": 375,
      "score": 85.90816,
      "score_before_time_limit": 85.90816,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9253333333333333,
      "interval90_width": 0.40466666666666656,
      "n": 375,
      "nrps": 0.08455155198666667,
      "score": 86.26650240066667,
      "score_before_time_limit": 86.26650240066667,
      "time_limit_failed_share": 0,
      "top_ece": 0.09505919999999997
     }
    },
    "score": 83.61830413355554
   },
   "capability": 66.1715118587882,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 799,606 input + 0 output tokens over 1,624 decisions = 492.368226601 tokens/decision. GPU-time cost USD 0.0078334333052483/1,000; old list-price cost USD 0.153865070812808/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0078334333052483/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.007833433305248303
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider-35b-a3b (Mapika)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 52.787614448504826,
    "I_sealed": 44.66182471953684,
    "base": 48.72471958402083,
    "excess": 5.552402086529744,
    "gap": 8.125789728967987,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 63.96752213183363,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 60.36489462094999,
       "judge": 60.44825313117996,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 43.862663906142174,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 56.52173913043479,
       "judge": 10.000000000000009,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 50.53265730753868,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 41.64121212121212,
       "hard": 47.127870370370395,
       "judge": 62.23203389830508,
       "standard": 44.23888888888891
      }
     },
     "sealed|choice": {
      "cc": 58.35475838263821,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 73.3944276106718,
       "hard": 49.60407239819003,
       "judge": 60.89385474860336,
       "standard": 64.52765118857003
      }
     },
     "sealed|noul": {
      "cc": 27.440215333904295,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48,
       "hard": 2.9126213592232997,
       "judge": 36.8421052631579,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 48.19050044206803,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 62.689858156028365,
       "hard": 43.85012254901962,
       "judge": 53.23419161676646,
       "standard": 42.05604060913702
      }
     }
    }
   },
   "jevbench_score": 66.50611544932869,
   "key": "decider-35b-a3b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9061371841155235,
    "decisive_rate": 0.7386666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.61830413355554,
     "cost": 34.38579876859529,
     "intelligence": 48.72471958402083,
     "speed": 91.27357492980725
    },
    "board_release": "v1.6.43",
    "capability": 66.1715118587882,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      18.69889165560131,
      26.594050353427875
     ],
     "B": [
      17.887370941224734,
      26.251970770447716
     ],
     "C": [
      12.985341427500913,
      20.369267436681472
     ]
    },
    "jevbench_score": 24.772887446577812,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 79,
    "rank": 112,
    "ranks": {
     "A": 112,
     "B": 109,
     "C": 127,
     "capability": 96
    },
    "scores": {
     "A": 24.772887446577812,
     "B": 24.135656537463312,
     "C": 17.20339406012348
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2493798840790987,
     "p50_s_raw": 0.04968994203954935,
     "p95_s_adjusted": 0.29907567272894087,
     "p95_s_raw": 0.07453783636447042
    },
    "views": {
     "Balanced 33:33:33": 22.248611212569305,
     "Emphasis on Accuracy 60:20:20": 22.101309301010737,
     "Emphasis on Cost 20:20:60": 18.91495293933302,
     "Emphasis on Speed 20:60:20": 27.229064334478497,
     "Intelligence only": 21.8839776409655
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 103,
   "ranked": true,
   "ranks": {
    "A": 103,
    "B": 105,
    "C": 110,
    "capability": 125
   },
   "repo": "https://huggingface.co/Mapika/decider-35b-a3b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 66.50611544932869,
    "B": 61.157070344582635,
    "C": 46.18480239536714
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.253978336090222,
    "p50_s_raw": 0.051989168045111,
    "p95_s_adjusted": 0.3008370825322345,
    "p95_s_raw": 0.07541854126611724
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-35B-A3B-Base with a trained decision readout, 34.7B parameters / 3B active",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.22742585943612,
      84.9038621148463
     ],
     "I": [
      43.19083908199794,
      52.63193908662482
     ],
     "capability": [
      61.40117742704215,
      67.98807044411026
     ],
     "composite": null,
     "seed": 27
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.61830413355554,
     "capability": 66.1715118587882,
     "intelligence": 48.72471958402083
    },
    "per_type": {
     "choice": {
      "accuracy": 0.692,
      "calibration": 78.68024999999997,
      "cc": 60.611198999649474,
      "cc_by_type": {
       "choice": 60.611198999649474
      },
      "cc_tiered": 61.16114025723592,
      "cc_tiered_open": 63.96752213183363,
      "cc_tiered_sealed": 58.35475838263821,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.611198999649474
     },
     "noul": {
      "accuracy": 0.6693333333333333,
      "calibration": 85.90816,
      "cc": 33.86666666666667,
      "cc_by_type": {
       "noul": 33.86666666666667
      },
      "cc_tiered": 35.65143962002323,
      "cc_tiered_open": 43.862663906142174,
      "cc_tiered_sealed": 27.440215333904295,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 33.86666666666667
     },
     "score": {
      "accuracy": 0.5893333333333334,
      "calibration": 86.26650240066667,
      "cc": 48.45208771929824,
      "cc_by_type": {
       "score": 48.45208771929824
      },
      "cc_tiered": 49.36157887480336,
      "cc_tiered_open": 50.53265730753868,
      "cc_tiered_sealed": 48.19050044206803,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 48.45208771929824
     }
    },
    "raw_on_A": {
     "calibration": 80.25053271222222,
     "capability": 65.45225190461375,
     "intelligence": 50.65397109700527
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.089448512005376,
    "Emphasis on Accuracy 60:20:20": 55.08110224368626,
    "Emphasis on Cost 20:20:60": 65.50488867938996,
    "Emphasis on Speed 20:60:20": 70.76915484585724,
    "Intelligence only": 46.27090954453351
   }
  },
  {
   "adapter_id": "images/decider-4b-v2/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mapika",
   "axes": {
    "calibration": 89.63735637833331,
    "cost": 79.08569750994965,
    "intelligence": 40.12597268917964,
    "speed": 90.31631787234045
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04618531250000016,
      "ece_score": 90.76293749999998,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 90.76293749999998,
      "score_before_time_limit": 90.76293749999998,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3573333333333333,
      "brier": 0.1263556454400002,
      "ece": 0.05272693333333338,
      "n": 375,
      "score": 89.45461333333333,
      "score_before_time_limit": 89.45461333333333,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9733333333333334,
      "interval90_width": 0.5584444444444441,
      "n": 375,
      "nrps": 0.09968243396666672,
      "score": 88.69451830166665,
      "score_before_time_limit": 88.69451830166665,
      "time_limit_failed_share": 0,
      "top_ece": 0.06321360000000002
     }
    },
    "score": 89.63735637833331
   },
   "capability": 64.88166453375648,
   "class": "system-one-open",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 822,885 input + 0 output tokens over 1,624 decisions = 506.702586207 tokens/decision. GPU-time cost USD 0.00497901479000153/1,000; old list-price cost USD 0.0152010775862069/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00497901479000153/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004979014790001528
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider-4b v2 (Mapika)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 45.24914817619414,
    "I_sealed": 35.00279720216514,
    "base": 40.12597268917964,
    "excess": 7.672963331590756,
    "gap": 10.246350974029,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 73.21703197407996,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 69.80182447310474,
       "judge": 76.26895187870797,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 26.461697722567287,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": -9.999999999999998,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 36.06871483193517,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.519696969696966,
       "hard": 36.743888888888875,
       "judge": 38.64050847457627,
       "standard": 24.635185185185193
      }
     },
     "sealed|choice": {
      "cc": 56.802191819905744,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 69.40359175227256,
       "hard": 50.62217194570135,
       "judge": 57.16945996275605,
       "standard": 62.31062938785566
      }
     },
     "sealed|noul": {
      "cc": 11.645451862940547,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48,
       "hard": -12.621359223300976,
       "judge": 23.684210526315795,
       "standard": 23.943661971830977
      }
     },
     "sealed|score": {
      "cc": 36.56074792364913,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 54.016382978723385,
       "hard": 31.53129901960784,
       "judge": 40.6503592814371,
       "standard": 31.75741116751262
      }
     }
    }
   },
   "jevbench_score": 43.08354714846196,
   "key": "decider-4b-v2",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (package and weights)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.921161825726141,
    "decisive_rate": 0.6426666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.63735637833331,
     "cost": 64.54376873042901,
     "intelligence": 40.12597268917964,
     "speed": 91.78549329698745
    },
    "board_release": "v1.6.43",
    "capability": 64.88166453375648,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 39,
    "capability_rank": 61,
    "composite_ci95": {
     "A": [
      28.727001690110686,
      49.92424966763833
     ],
     "B": [
      25.10971569724804,
      45.24030143351226
     ],
     "C": [
      19.949306729243542,
      34.66961782474885
     ]
    },
    "jevbench_score": 41.240894350857424,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 53,
    "rank": 80,
    "ranks": {
     "A": 80,
     "B": 83,
     "C": 84,
     "capability": 103
    },
    "scores": {
     "A": 41.240894350857424,
     "B": 36.84955227461344,
     "C": 28.639509965873213
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19992743488401174,
     "p50_s_raw": 0.024963717442005873,
     "p95_s_adjusted": 0.3315723263658583,
     "p95_s_raw": 0.09078616318292916
    },
    "views": {
     "Balanced 33:33:33": 37.65578397809399,
     "Emphasis on Accuracy 60:20:20": 31.83485819738892,
     "Emphasis on Cost 20:20:60": 39.12905962819718,
     "Emphasis on Speed 20:60:20": 44.051943214406094,
     "Intelligence only": 25.842630080539234
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 139,
   "ranked": true,
   "ranks": {
    "A": 139,
    "B": 146,
    "C": 144,
    "capability": 131
   },
   "repo": "https://github.com/Mapika/decider",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 43.08354714846196,
    "B": 38.01165070879777,
    "C": 29.9191299642097
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20652835872024297,
    "p50_s_raw": 0.02826417936012149,
    "p95_s_adjusted": 0.45018240716308344,
    "p95_s_raw": 0.15009120358154174
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.18439184191017,
      90.03628745741203
     ],
     "I": [
      34.68576750629042,
      43.56524556183246
     ],
     "capability": [
      60.38851961863991,
      66.00729141285105
     ],
     "composite": null,
     "seed": 28
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.63735637833331,
     "capability": 64.88166453375648,
     "intelligence": 40.12597268917964
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6933333333333334,
      "calibration": 90.76293749999998,
      "cc": 60.781713289694295,
      "cc_by_type": {
       "choice": 60.781713289694295
      },
      "cc_tiered": 65.00961189699285,
      "cc_tiered_open": 73.21703197407996,
      "cc_tiered_sealed": 56.802191819905744,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.781713289694295
     },
     "noul": {
      "accuracy": 0.592,
      "calibration": 89.45461333333333,
      "cc": 18.399999999999995,
      "cc_by_type": {
       "noul": 18.399999999999995
      },
      "cc_tiered": 19.053574792753917,
      "cc_tiered_open": 26.461697722567287,
      "cc_tiered_sealed": 11.645451862940547,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.399999999999995
     },
     "score": {
      "accuracy": 0.512,
      "calibration": 88.69451830166665,
      "cc": 36.877482456140385,
      "cc_by_type": {
       "score": 36.877482456140385
      },
      "cc_tiered": 36.31473137779215,
      "cc_tiered_open": 36.06871483193517,
      "cc_tiered_sealed": 36.56074792364913,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.877482456140385
     }
    },
    "raw_on_A": {
     "calibration": 88.78997857788461,
     "capability": 66.11763148027418,
     "intelligence": 43.44528438266377
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 39.72415820193711,
    "Emphasis on Accuracy 60:20:20": 32.69848175118202,
    "Emphasis on Cost 20:20:60": 43.558871780466724,
    "Emphasis on Speed 20:60:20": 45.4940438720906,
    "Intelligence only": 25.842630080539234
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mapika",
   "axes": {
    "calibration": 82.34718405098182,
    "cost": 56.10708393368468,
    "intelligence": 58.54290392078144,
    "speed": 85.76268320171138
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13017175324675329,
      "ece_score": 73.96564935064934,
      "mean_tvd": null,
      "n": 738,
      "n_ece_hard": 308,
      "n_tvd": 0,
      "score": 73.37392415584415,
      "score_before_time_limit": 73.96564935064934,
      "time_limit_failed_share": 0.008
     },
     "noul": {
      "abstention_rate": 0.1967654986522911,
      "brier": 0.06013664803234495,
      "ece": 0.0693757412398923,
      "n": 371,
      "score": 86.12485175202154,
      "score_before_time_limit": 86.12485175202154,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.986449864498645,
      "interval90_width": 0.640469738030713,
      "n": 369,
      "nrps": 0.06538499443315268,
      "score": 87.54277624507979,
      "score_before_time_limit": 87.77684783931798,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.0895390243902439
     }
    },
    "score": 82.34718405098182
   },
   "capability": 70.44504398588163,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 717,195 input + 0 output tokens over 1,477 decisions = 485.57549086 tokens/decision. GPU-time cost USD 0.0290467269498916/1,000; old list-price cost USD 0.0437017941773866/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0290467269498916/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.04500444
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.029046726949891646
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 7,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decider chat on Gemma-4-31B-it (Mapika, frozen base, inference technique)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 55.72975554572549,
    "I_sealed": 61.356052295837394,
    "base": 58.54290392078144,
    "excess": -8.199684392550148,
    "gap": -5.626296750111905,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 76.96981695560446,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 75.46398238439761,
       "judge": 72.31377719182596,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 43.846790890269155,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 46.372658791302854,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.266060606060606,
       "hard": 49.83425925925926,
       "judge": 51.426101694915246,
       "standard": 28.422592592592586
      }
     },
     "sealed|choice": {
      "cc": 82.26456829463379,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 90.68804966373514,
       "hard": 73.02036199095022,
       "judge": 88.8268156424581,
       "standard": 86.69786919571378
      }
     },
     "sealed|noul": {
      "cc": 54.93243610873211,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 37.86407766990292,
       "judge": 63.1578947368421,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 46.871152484146286,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 60.35531914893617,
       "hard": 42.373504901960835,
       "judge": 52.318562874251484,
       "standard": 40.95324873096443
      }
     }
    }
   },
   "jevbench_score": 68.13342031667979,
   "key": "decider-chat-gemma4-31b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475 + Mapika/decider-chat-gemma4-31b@22975cbd1716170264a1d2a7e1105787e08c5d95",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9765100671140939,
    "decisive_rate": 0.8032345013477089,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.62244964732962,
     "cost": 50.78502198104071,
     "intelligence": 58.54290392078144,
     "speed": 85.78163968405963
    },
    "board_release": "v1.6.43",
    "capability": 70.58267678405554,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 25,
    "capability_rank": 44,
    "composite_ci95": {
     "A": [
      63.831274751690664,
      67.00563695142573
     ],
     "B": [
      61.40092144952185,
      66.08555760610543
     ],
     "C": [
      50.02666080334015,
      67.00563695142573
     ]
    },
    "jevbench_score": 66.07956202755186,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 25,
    "rank": 46,
    "ranks": {
     "A": 46,
     "B": 44,
     "C": 39,
     "capability": 75
    },
    "scores": {
     "A": 66.07956202755186,
     "B": 64.42088721187244,
     "C": 62.90905728971202
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4011345919920132,
     "p50_s_raw": 0.12556729599600658,
     "p95_s_adjusted": 0.658485045604175,
     "p95_s_raw": 0.2542425228020875
    },
    "views": {
     "Balanced 33:33:33": 61.94527894229558,
     "Emphasis on Accuracy 60:20:20": 60.537949991324034,
     "Emphasis on Cost 20:20:60": 56.94013011167525,
     "Emphasis on Speed 20:60:20": 69.69141016272579,
     "Intelligence only": 58.54290392078144
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 94,
   "ranked": true,
   "ranks": {
    "A": 94,
    "B": 87,
    "C": 73,
    "capability": 96
   },
   "repo": "https://github.com/Mapika/decider",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.13342031667979,
    "B": 65.97191254525063,
    "C": 64.86437122962322
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.40184165698010477,
    "p50_s_raw": 0.12592082849005237,
    "p95_s_adjusted": 0.6602018301025965,
    "p95_s_raw": 0.2551009150512982
   },
   "status": {
    "answered_ok": 1478,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-31B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.49144818930593,
      82.08290139197796
     ],
     "I": [
      53.001527166194506,
      62.9130738579859
     ],
     "capability": [
      66.47453104990954,
      71.637319105497
     ],
     "composite": null,
     "seed": 30
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.34718405098182,
     "capability": 70.44504398588163,
     "intelligence": 58.54290392078144
    },
    "per_type": {
     "choice": {
      "accuracy": 0.856,
      "calibration": 73.37392415584415,
      "cc": 81.5844566751608,
      "cc_by_type": {
       "choice": 81.5844566751608
      },
      "cc_tiered": 79.61719262511912,
      "cc_tiered_open": 76.96981695560446,
      "cc_tiered_sealed": 82.26456829463379,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.5844566751608
     },
     "noul": {
      "accuracy": 0.776,
      "calibration": 86.12485175202154,
      "cc": 55.2,
      "cc_by_type": {
       "noul": 55.2
      },
      "cc_tiered": 49.38961349950063,
      "cc_tiered_open": 43.846790890269155,
      "cc_tiered_sealed": 54.93243610873211,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 55.2
     },
     "score": {
      "accuracy": 0.6613333333333333,
      "calibration": 87.54277624507979,
      "cc": 46.969219298245605,
      "cc_by_type": {
       "score": 46.969219298245605
      },
      "cc_tiered": 46.621905637724566,
      "cc_tiered_open": 46.372658791302854,
      "cc_tiered_sealed": 46.871152484146286,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 46.969219298245605
     }
    },
    "raw_on_A": {
     "calibration": 80.29664856797598,
     "capability": 70.99028048840852,
     "intelligence": 61.68391240884105
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.42657769989975,
    "Emphasis on Accuracy 60:20:20": 61.93667595682244,
    "Emphasis on Cost 20:20:60": 60.819291886820835,
    "Emphasis on Speed 20:60:20": 71.54631649549482,
    "Intelligence only": 58.54290392078144
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "tachara-ai (Hugging Face)",
   "axes": {
    "calibration": 88.61282851120822,
    "cost": 66.96709724088544,
    "intelligence": 62.78089467434111,
    "speed": 90.3121994607612
   },
   "base_model": "google/gemma-4-12B-it",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03870305643842932,
      "ece_score": 92.25938871231413,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 92.25938871231413,
      "score_before_time_limit": 92.25938871231413,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.10394989403343095,
      "ece": 0.09193095805447535,
      "n": 373,
      "score": 81.61380838910492,
      "score_before_time_limit": 81.61380838910492,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.5199372759856628,
      "n": 372,
      "nrps": 0.08852525470256986,
      "score": 91.96528843220563,
      "score_before_time_limit": 91.96528843220563,
      "time_limit_failed_share": 0,
      "top_ece": 0.03608448832665872
     }
    },
    "score": 88.61282851120822
   },
   "capability": 75.69686159277467,
   "code_revision": "THE AUTHOR'S OWN SERVER, run unmodified: aminry/decisio at 4f849ff2dbae10a53026e382ce3dd60b95780594 (one commit after tag v0.32.0 e7e8937f35b0611d327295563b0cffbeb8bafdc3; the commit changes README.md only, so every src/ file equals the tag; package decisio 0.32.0), installed on the pod with the author's own 'uv sync --extra serve --frozen' from the repository's uv.lock (sha256 d82e40ba4b3693a17e7cd85f17604d1f818fae6dad431293e3b4541932bbbbe3) with credentials absent, before any sealed byte; the sealed run itself had no network. Entrypoint: python -m decisio.serve.vllm_engine --base /model --max-model-len 32768 --host 127.0.0.1 --port 8000, every other flag at its 0.32.0 default (over-limit refuse, header-limit auto, concurrency serial, orders 1, think off, no adapters, no branch log, no task store, stock vLLM with no patches), HF offline flags set. The server's native endpoint POST /v1/systemone was used with no response shim and no launcher shim; the request path is the frozen v1.6 measurement driver with the frozen TypeSafe adapter. The only flag outside the author's defaults, --max-model-len 32768, fixes the context length: without it 0.32.0 picks an automatic length up to 65,536 that depends on the model card, which would make the refusal set card-dependent. On-pod asserts passed before any sealed input: distribution versions decisio 0.32.0, vllm 0.31.0, torch 2.13.0, transformers 5.17.0, triton 3.7.1 (tokenizers 0.23.2 in uv.lock; python 3.12); /health flavor report {name commit, served_name Decisio-12b-tuned-commit, overrides {}, pending []}; abstention_tasks {} (no task source configured, so no per-task calibration, head or abstention applies). Source review (line by line over the request path, before measurement): no branch on item text, ids, gold labels or item lists; the disclosed item-blind label-shape rules do inspect option keys/descriptions. No network, subprocess, eval or pickle on the request path; no stop condition. Model: tachara-ai/Decisio-12b-tuned-commit at 245e00dc45abad14d693b1ebae4d4c7aa9cf72f1, a MERGED BF16 checkpoint (6 safetensors shards, 23,919,548,376 shard bytes; every file hashed against the repository SHA256SUMS on the node) of google/gemma-4-12B-it at 707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7 plus a LoRA with adapter dimension 16 merged in fp32 and rounded once to bf16 (stochastic, seed 0, per the model card); on one CUDA device, no quantisation. Author code and weights were fetched by our code at the pinned revisions and ran only on a disposable rented GPU VM; the measurement kit was hash-pinned and independently reviewed before measurement, and the draw was used once for this row. Weights: 12B-class merged checkpoint, not an adapter.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 831,272 input + 1,485 output tokens over 1,485 decisions = 560.779124579 tokens/decision. GPU-time cost USD 0.0126210869476997/1,000; old list-price cost USD 0.0281389562289562/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0126210869476997/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.027857566666666674
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.012621086947699698
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of Decisio-12b-tuned-commit in this intake (our own intake). A NEW ROW, A VARIANT OF THE DECISIO FAMILY: tachara-ai/Decisio-12b-tuned-commit is a different set of weights from the live decisio-gemma-4-12b-v080 and -v090 rows, which are the stock google/gemma-4-12B-it served by decisio; none of its 6 shard LFS sha256s equals the safetensors blob of google/gemma-4-12B-it at 707f0a3b, of google/gemma-4-31B-it at 842da379, of the decisio FP8 31B repository at eaf1e4fa, or of the weights behind the live decisio rows (byte-identity evidence only, not proof of independent training). It replaces and supersedes none of the live decisio rows. The model card states that a sibling repository, Decisio-12b-tuned-calibrated, holds the same weights and differs in decision_config.json only; that sibling was not measured and nothing is inferred about it. The name is 'Jev-compatible' only: it is a Decisio model, not a Jev model. FINE-TUNE OF GEMMA 4: a LoRA merged into google/gemma-4-12B-it (see code_revision). AUTHOR'S OWN SERVER on our rented hardware, run unmodified (see code_revision) with no shim. NATIVE OUTPUTS: probabilities for choice, noul and score from the option-letter readout (vLLM max_tokens=1 for the option-label readout, softmax over the label tokens, think off; state-prefix registration can add a warm-up, excluded from native usage) at the repository's shipped per-type temperatures (yes/no 3.314, score 2.1109, choice 2-10 options 2.1317, 11-26 options 1.8766, 27-255 options 1.7181); no generated explanation. CONFIGURATION: the repository's shipped flavor 'commit', exactly as shipped; the card also names the calibrated sibling. We did not measure any other flavor or setting and infer nothing about one. JEVBENCH-AWARE DEFAULT, THE COMMIT RULE (a disclosed non-raw, item-blind output transform, treated like the DOPEST 0.801 floor): for yes/no questions, a temperature-scaled P(yes) strictly between 0.20 and 0.80 is reported at the nearer band edge, 0.80 if it is above 0.5, otherwise 0.20 (exactly 0.5 goes to 0.20). The band is exactly JevBench's own yes/no decision band: JevBench scores an answer strictly between 0.20 and 0.80 as an abstention, counted wrong, and the author's repository ships its own reading of that open-set rule (not imported by the server). On this draw the rule moved 193 of the 373 answered yes/no answers to a band edge, so the row has no yes/no abstentions; without it those answers would be scored as abstentions. It is the shipped flavor's default, measured as shipped. The argmax never changes; the Calibration axis includes the effect of this transform, and we did not measure a variant without it. LABEL-SHAPE RULES ON BY DEFAULT: hide_index_keys, describe_options and desnake_labels. The author documents hide_index_keys as motivated by the Decision Index option_N format and desnake_labels by BANKING77; they act on the shape of the labels of any request, never on item text, ids, gold labels or item lists (source review confirmed item-blind), and the answer keys are always the request's own keys. The frozen offline layout counter recorded 739 describe_options, 375 noul_as_letters and 11 hide_index_keys applications across all 1,500 draw rows, including the refused rows; desnake_labels recorded 0. Of 375 yes/no rows, 373 were answered. On the 231 public items the corresponding counts were 139 / 74 / 0 (desnake_labels 0). TRAINING DATA (author statement, not independently verified): the model card says the training data is not disclosed, that no Decision Index or JevBench evaluation items were used in training or checkpoint selection, that the data was decontaminated against both boards' public test sets, and that official training splits of some public benchmarks were used under their licences. We did not inspect the training examples or check overlap with JevBench items; contamination of the sealed draw is unchecked. The card says the temperatures were fitted by minimum log loss per question type and choice band on the plain readout; it does not identify the data of that fit. LICENCE NOTE: Apache-2.0 per the model card, LICENSE and NOTICE on the weights and per the author server licence; the base is Gemma 4 under its own terms. CONTEXT AND REFUSALS: the server refuses a row natively when even the plain layout (system turn, chat template, state, question, no header) exceeds its 32,767-token prompt limit (32,767 answered, 32,768 refused; boundary proven offline, and a row whose header alone pushes it over is answered on the plain layout instead); a refused row is never truncated. NOT ANSWERED, counted wrong: the draw's 15 standard long positions (plain-layout prompts of 76,531 to 81,726 tokens by our offline count), counted and frozen in the preregistered refusal list before launch; all returned HTTP 422 and the measured 422 set equals the preregistered set exactly (no unexpected failure; every other row HTTP 200). ANSWERED: 1,485 of 1,500. 10 s RULE: 0 decisions over 10 s, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); measured loopback median 0.058397072 s, interpolated p95 0.099410957 s (upper-order p95 0.099450955 s), maximum 0.167558985 s over all 1,500 requests on the recorded hardware. The v1.7 share view uses the common self-hosted latency estimate, 2 times measured p50 plus 0.15 s = 0.266794144 s; this adjustment is an assumption, not a measured hosted endpoint. USAGE: the server-native input usage (715,443 tokens) equals our first-party count exactly, 0 mismatches; output_tokens 1 per answered row natively (1,485 in total). DETERMINISM (record-only): the first 50 requests were run twice; 3 of 50 were bit-identical, 0 argmax flips, maximum probability difference 0.0195 (observed on the serial vLLM path; its cause was not isolated; a short smoke probe also varied between 0.8451876 and 0.8500589). The scored rows are the first run and stand. ATTEMPT HISTORY: one attempt, no earlier failed or discarded attempt on the sealed input; the pre-sealed smoke and gates passed before any sealed item.",
   "display": "Decisio 12B tuned commit (variant of the Decisio family)",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-25",
    "release": "v1.7.6",
    "results_sha256": "456546008cd5759995a21c706181c70926f81d32944a7eda4f3367fd8f800841"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB, driver 580.178.04), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.558095890410959/h recorded by the runner, 150 GB disk. One VM for this row, one attempt. Measured 25 Oct 2026, VM window about 23:26-23:39Z, about USD 0.55 GPU rental (elapsed time at the recorded hourly rate). Author server at 127.0.0.1:8000, network cut before sealed input (offline proof and cut proof receipts), read-only model and input, private output, HF offline flags, no credentials. Before sealed input: model files re-hashed against SHA256SUMS; the dependency Triton gate, a pre-warm and the smoke passed (the smoke includes both 422 over-limit probes at 32,768 and 80k tokens); the TTL fit gate admitted the run. Input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b (v1.6-regular-20261012) was re-checked inside the container (match). Complete 1,500-row run, rc 0. Raw output sha256 e9c889e1a138c875eb52521748e63b4344043c7a5598527c073172dd1c81594f (the pull-hash receipt). Node-shred and local-shred receipts verified; exact-id teardown receipt confirms the VM closed. GPU rental is separate from the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 68.7111277698651,
    "I_sealed": 56.85066157881713,
    "base": 62.78089467434111,
    "excess": 2.020939420692258,
    "gap": 11.86046619104797,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 73.51239714334199,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 62.25228059138094,
       "judge": 68.35860250494397,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 80.3768115942029,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 80,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 52.24417457205039,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.52839650119154,
       "hard": 47.74971241071595,
       "judge": 52.83373682048644,
       "standard": 57.20664455749461
      }
     },
     "sealed|choice": {
      "cc": 64.43982160940423,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 86.6581956797967,
       "hard": 57.84665201804093,
       "judge": 58.570029382957905,
       "standard": 75.32166209660409
      }
     },
     "sealed|noul": {
      "cc": 70.28427671145118,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 72.41379310344827,
       "hard": 65.45454545454545,
       "judge": 66.15384615384616,
       "standard": 85.07462686567165
      }
     },
     "sealed|score": {
      "cc": 35.82788641559598,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 53.92109493878885,
       "hard": 28.30185867749081,
       "judge": 36.085638239942895,
       "standard": 41.4467098936895
      }
     }
    }
   },
   "jevbench_score": 75.16131305837865,
   "key": "decisio-gemma-4-12b-tuned-commit",
   "last_measured_on": "2026-10-25",
   "licence": "Apache-2.0 per the model card, LICENSE and NOTICE on the weights and the author server; base google/gemma-4-12B-it public, ungated, under its own terms",
   "listing": "ranked",
   "measured_in": "v1.7.6",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "tachara-ai/Decisio-12b-tuned-commit@245e00dc45abad14d693b1ebae4d4c7aa9cf72f1 + aminry/decisio@4f849ff2dbae10a53026e382ce3dd60b95780594",
   "new_in": "v1.7.6",
   "noul_decisive": {
    "acc_among_decisive": 0.871313672922252,
    "decisive": 373,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 17,
   "ranked": true,
   "ranks": {
    "A": 17,
    "B": 34,
    "C": 16,
    "capability": 60
   },
   "repo": "https://huggingface.co/tachara-ai/Decisio-12b-tuned-commit",
   "scores": {
    "A": 75.16131305837865,
    "B": 72.30942342000414,
    "C": 75.16131305837865
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26679414399998225,
    "p50_s_raw": 0.05839707199999111,
    "p95_s_adjusted": 0.3488219143000151,
    "p95_s_raw": 0.09941095715000756
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 67.26883523733275,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 35.13983904533158
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 67.26883523733275
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 61.316515410629954,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": 92,
        "score": 42.85092481089304
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.316515410629954
      },
      "composite_gen": {
       "accuracy": 0.7326732673267327,
       "cc": 55.20486090453259,
       "cc_by_type": {
        "choice": 70.73170731707317,
        "noul": 55.55555555555556,
        "score": 39.327319840969025
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 55.20486090453259
      },
      "fairness": {
       "accuracy": 0.8541666666666666,
       "cc": 84.67262300350875,
       "cc_by_type": {
        "choice": 100,
        "noul": 86.66666666666667,
        "score": 67.35120234385954
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.67262300350875
      },
      "jaggedness": {
       "accuracy": 0.6333333333333333,
       "cc": 48.38264830927233,
       "cc_by_type": {
        "choice": 58.132820019249294,
        "noul": 67.56756756756756,
        "score": 19.44755734100011
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 48.38264830927233
      },
      "llm_router": {
       "accuracy": 0.8274760383386581,
       "cc": 72.14922012688827,
       "cc_by_type": {
        "choice": 73.84872765695167,
        "noul": 89.69072164948453,
        "score": 52.90821107422858
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 72.14922012688827
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 45.73235129034188,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 75,
        "score": 11.68158995349986
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 45.73235129034188
      },
      "multilingual": {
       "accuracy": 0.7272727272727273,
       "cc": 59.84482702135487,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": 66.66666666666667,
        "score": 47.10322359205068
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 59.84482702135487
      },
      "negation_pair_gen": {
       "accuracy": 0.8023255813953488,
       "cc": 49.24476329337568,
       "cc_by_type": {
        "choice": 90,
        "noul": 46.66666666666666,
        "score": 11.067623213460354
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 49.24476329337568
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 75.35377046178358,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100,
        "score": 47.08233240637176
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 75.35377046178358
      },
      "usecase": {
       "accuracy": 0.6590909090909091,
       "cc": 55.50649874544674,
       "cc_by_type": {
        "choice": 58.33333333333336,
        "noul": 66.66666666666667,
        "score": 41.51949623634018
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 55.50649874544674
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 38.82047887275462,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100,
        "score": -45.44332528649804
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 38.82047887275462
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 44.71318819900765,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 21.543558452629696
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 44.71318819900765
      },
      "de": {
       "accuracy": 0.875,
       "cc": 83.22813715774681,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 73.57821678297495
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.22813715774681
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 71.33436068920314,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 66.66666666666667,
        "score": 97.61574500988128
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 71.33436068920314
      },
      "en": {
       "accuracy": 0.7324066029539531,
       "cc": 59.691544815211365,
       "cc_by_type": {
        "choice": 67.2691326686274,
        "noul": 73.38129496402878,
        "score": 38.42420681297789
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 59.691544815211365
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 69.82090482784348,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 29.03857092724983
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 69.82090482784348
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 86.05624760972051,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": 90.95562807506322
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 86.05624760972051
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 73.60280835996384,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 63.16136625636213
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 73.60280835996384
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 55.66558052274937,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 75,
        "score": 25.516294640873795
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 55.66558052274937
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.8,
       "cc": 55.24199041446298,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -20.244410673992185
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 55.24199041446298
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 51.625121154190175,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 80,
        "score": 32.06360810117259
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 51.625121154190175
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 33.92524256687532,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 46.22017214507041
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 33.92524256687532
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.02353869768989,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.07061609306963
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.02353869768989
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 32.7221653981354,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 16.781214809124823
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 32.7221653981354
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 61.07657575616964,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 60.00000000000001,
        "score": 46.08203365869091
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 61.07657575616964
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 66.05025592954213,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 40.57858157766589
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 66.05025592954213
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 24.57605603447779,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 30.323912784284435
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 24.57605603447779
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 48.73848556245495,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50,
        "score": 17.95458712214747
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 48.73848556245495
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 63.26031326555928,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": 100,
        "score": 86.55513334506495
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 63.26031326555928
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 46.487524666943465,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 6.129240667497049
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.487524666943465
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7453333333333333,
       "cc": 67.10614439618055,
       "cc_by_type": {
        "choice": 67.10614439618055
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 67.10614439618055
      },
      "noul": {
       "accuracy": 0.8666666666666667,
       "cc": 73.33333333333334,
       "cc_by_type": {
        "noul": 73.33333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 73.33333333333334
      },
      "score": {
       "accuracy": 0.5786666666666667,
       "cc": 39.703122087156686,
       "cc_by_type": {
        "score": 39.703122087156686
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 39.703122087156686
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 15.347146722143526,
       "cc_by_type": {
        "choice": 20,
        "noul": 25,
        "score": 1.041440166430574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 15.347146722143526
      },
      "code_linting": {
       "accuracy": 0.6666666666666666,
       "cc": 51.60783601808881,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 77.77777777777777,
        "score": -4.320729350840535
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 51.60783601808881
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 65.38801499319703,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 28.684370182843132
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 65.38801499319703
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 16.537880084637816,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -13.544254482928642
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 16.537880084637816
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 52.516672769135546,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 67.69494584363854
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 52.516672769135546
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": 34.81929131567247,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 33.33333333333333,
        "score": 34.01762237469037
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 34.81929131567247
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 58.91679800241892,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 71.42857142857142,
        "score": 44.53750885319516
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 58.91679800241892
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 47.053185149789385,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": 50,
        "score": 53.877325484211354
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 47.053185149789385
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 34.24280870741969,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 60.00000000000001,
        "score": 26.061759455592394
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 34.24280870741969
      },
      "knowledge_graphs": {
       "accuracy": 0.6818181818181818,
       "cc": 55.44950237560722,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100,
        "score": 4.892404343095757
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 55.44950237560722
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 61.10411665841116,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 17.162479174199895
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 61.10411665841116
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 11.937064584279774,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -0.7479460321069054
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 11.937064584279774
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 68.22947053888926,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 56.20472639593841
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.22947053888926
      },
      "model_routing": {
       "accuracy": 0.8571428571428571,
       "cc": 74.96732105007766,
       "cc_by_type": {
        "choice": 83.63636363636363,
        "noul": 100,
        "score": 41.265599513869354
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.96732105007766
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 15.725784823047766,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 58.693669248413926
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 15.725784823047766
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 47.98819341018694,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 44.05743074123863
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 47.98819341018694
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 42.66737091740362,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 50,
        "score": 22.337088121669012
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 42.66737091740362
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 37.21267435688048,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": 66.66666666666667,
        "score": 31.927878143105172
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 37.21267435688048
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.01193249489371,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 92.71195406831106
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.01193249489371
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.4228400017454,
      89.51286151618956
     ],
     "I": [
      59.205525821419656,
      66.08910669761744
     ],
     "capability": [
      71.9959068772358,
      77.02677744787904
     ],
     "composite": null,
     "seed": 28
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.61282851120822,
     "capability": 75.69686159277467,
     "intelligence": 62.78089467434111
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7453333333333333,
      "calibration": 92.25938871231413,
      "cc": 67.10614439618055,
      "cc_by_type": {
       "choice": 67.10614439618055
      },
      "cc_tiered": 68.9761093763731,
      "cc_tiered_open": 73.51239714334199,
      "cc_tiered_sealed": 64.43982160940423,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.10614439618055
     },
     "noul": {
      "accuracy": 0.8666666666666667,
      "calibration": 81.61380838910492,
      "cc": 73.33333333333334,
      "cc_by_type": {
       "noul": 73.33333333333334
      },
      "cc_tiered": 75.33054415282704,
      "cc_tiered_open": 80.3768115942029,
      "cc_tiered_sealed": 70.28427671145118,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 73.33333333333334
     },
     "score": {
      "accuracy": 0.5786666666666667,
      "calibration": 91.96528843220563,
      "cc": 39.703122087156686,
      "cc_by_type": {
       "score": 39.703122087156686
      },
      "cc_tiered": 44.03603049382318,
      "cc_tiered_open": 52.24417457205039,
      "cc_tiered_sealed": 35.82788641559598,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 39.703122087156686
     }
    },
    "raw_on_A": {
     "calibration": 84.59655678674739,
     "capability": 74.22390984881723,
     "intelligence": 63.85126291088707
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 71.54129959391963,
    "Emphasis on Accuracy 60:20:20": 67.75926614828857,
    "Emphasis on Cost 20:20:60": 69.63862688401194,
    "Emphasis on Speed 20:60:20": 78.02841216697239,
    "Intelligence only": 62.78089467434112
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "tachara-ai (Hugging Face)",
   "axes": {
    "calibration": 88.68433331756145,
    "cost": 70.73054063499694,
    "intelligence": 68.62145857576381,
    "speed": 90.60515121372691
   },
   "base_model": "google/gemma-4-12B-it",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06157771278323828,
      "ece_score": 87.68445744335234,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 354,
      "n_tvd": 0,
      "score": 87.68445744335234,
      "score_before_time_limit": 87.68445744335234,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.09098459442278795,
      "ece": 0.04205621260120893,
      "n": 375,
      "score": 91.58875747975821,
      "score_before_time_limit": 91.58875747975821,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9946666666666667,
      "interval90_width": 0.4128888888888887,
      "n": 375,
      "nrps": 0.05156355420808304,
      "score": 86.77978502957379,
      "score_before_time_limit": 86.77978502957379,
      "time_limit_failed_share": 0,
      "top_ece": 0.10642037260022068
     }
    },
    "score": 88.68433331756145
   },
   "capability": 78.65289594666262,
   "code_revision": "THE AUTHOR'S OWN SERVER, run unmodified: aminry/decisio at tag v0.35.0 (commit 3d6098df654a854d9e226cfdf61f083eb1b15b04; the source equals the PyPI wheel decisio 0.35.0, wheel sha256 34a084ee10e001c6a598ff591a8e44bcddc23712bac0b7f97087dea23ad5dbb5, sdist sha256 2790b2127dd936ab5a279825aad9b4e20dd6742dd74000fbbbae2b6d8c551d0d), installed on the disposable rented pod with the author's own 'uv sync --extra serve --frozen' against uv.lock sha256 2e52080c7791e8104c41dfa38d72fc52084654c381668c53731775c9214eff0c with credentials absent, before any sealed byte; the sealed run itself had no network. Entrypoint: python -m decisio.serve.vllm_engine --base /model --max-model-len 65536 --host 127.0.0.1 --port 8000, every other flag at its 0.35.0 default (over-limit truncate at 45,000 tokens, think on with factor 0.495, budget 256, choice questions only, header-limit auto, concurrency serial, orders 1, no adapters, no branch log, no task store, stock vLLM with no patches), HF offline flags set. The server's native endpoint POST /v1/systemone was used with no response shim and no launcher shim; the request path is the frozen v1.6 measurement driver with the frozen TypeSafe adapter. The only flag outside the author's defaults, --max-model-len 65536, is the 12B context ceiling and what the author's flag-less command picks on a card that fits it; it makes the cut rows fit independently of the card (at the 32,768 setting of the sibling row every 45,000-token cut read would be refused). On-pod asserts passed before any sealed input: distribution versions decisio 0.35.0, vllm 0.31.0, torch 2.13.0, transformers 5.17.0, triton 3.7.1 (tokenizers 0.23.2 in uv.lock; python 3.12); /health on the scored process before and after the run: max_model_len 65,536, over_limit truncate / prompt_limit 45,000, think default true / factor 0.495 / budget 256 / types [choice], flavor commit-think, served_name Decisio-12b-tuned-commit-think, overrides {}, pending []. Source review (line by line over the request path and the 0.32.0 to 0.35.0 delta): no branch on item text, ids, gold labels or item lists; the disclosed item-blind label-shape rules do inspect option keys/descriptions; no network, subprocess, eval or pickle on the request path; no stop condition. Model: tachara-ai/Decisio-12b-tuned-commit-think at e36aeb243dea3008c593f454112840a1a29e5837, a MERGED BF16 checkpoint (20 files, 6 safetensors shards, 23,919,548,376 shard bytes; every file hashed against the repository SHA256SUMS on the node and again inside the run container) of google/gemma-4-12B-it at 707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7 plus a LoRA with adapter dimension 16 merged in fp32 and rounded once to bf16 (stochastic, seed 0, per the model card); on one CUDA device, no quantisation. Author code and weights were fetched by our code at the pinned revisions and ran only on a disposable rented GPU VM; the measurement kit was hash-pinned (ceaf8953) and independently reviewed (PASS) before measurement. Weights: 12B-class merged checkpoint, not an adapter.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 616,406 input + 9,110 output tokens over 1,489 decisions = 420.091336467 tokens/decision. GPU-time cost USD 0.0094547194272077/1,000; old list-price cost USD 0.0216163868368032/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0094547194272077/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.02904646666666667
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009454719427207697
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of Decisio-12b-tuned-commit-think. DRAW: measured on the 27 Oct 2026 regular draw (fresh sealed items; not yet equated to the reference scale, so its numbers are shown as published, offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own draw. A NEW ROW OF THE DECISIO FAMILY AND A SERVING-CONFIGURATION VARIANT OF A LIVE ROW: the weights tachara-ai/Decisio-12b-tuned-commit-think@e36aeb24 are the same weights as the live Decisio 12B tuned commit row (tachara-ai/Decisio-12b-tuned-commit@245e00dc): 16 of 20 files are identical by LFS sha256 or git blob oid and by sha256, including all six safetensors shards, the index, tokenizer, config and chat template; exactly 4 files differ (decision_config.json, README.md, NOTICE, SHA256SUMS). The behavioural difference is decision_config.json only: decisio 0.31.0 to 0.35.0, a think default {factor 0.495, budget 256, types [choice]}, over_limit truncate at 45,000 tokens, and the flavor name commit to commit-think. It is a separate row and replaces none of the live rows; nothing is inferred about the author's third same-weights repository (Decisio-12b-tuned-calibrated), which was not measured. The name is 'Jev-compatible' only: it is a Decisio model, not a Jev model. AUTHOR'S OWN SERVER on our rented hardware, run unmodified (see code_revision), no shim. NATIVE OUTPUTS: probabilities for choice, noul and score from the option-letter readout (one vLLM label-logprob readout, softmax over the label tokens) at the repository's shipped per-type temperatures (yes/no 3.314, score 2.1109, choice 2-10 options 2.1317, 11-26 options 1.8766, 27-255 options 1.7181). CONFIGURATION: the repository's shipped flavor 'commit-think', exactly as shipped; we measured no other flavor or setting and infer nothing about one. THINK PASS (author's feature, on by default in this flavor): a choice question of at most 16 options whose served first-read top probability is below 0.37125 (0.75 x 0.495) is read a second time after up to 256 greedy reasoning tokens; the two reads are mixed in log space and served at the same temperature. Noul and score never think and a cut state never thinks. OBSERVED: 36 of the 746 think-eligible choice decisions thought, 7,621 reasoning tokens in total, 19 of them stopped at the 256-token budget; no noul, score or cut decision thought. TRUNCATION (author's over-limit truncate): a state whose rendered text exceeds the cut room (45,000 tokens minus the longest plain user turn minus 32) is cut to its START at a whitespace boundary within 64 tokens, with no marker, and answered by the single read without thinking. Exactly the 5 pre-registered long items (76,554 to 81,413 tokens before the cut) were cut to 44,801 to 44,816 tokens; the observed cut set equals the counted set. Their answers rest on the first part of the state only. They are answered, not refused, and counted as answers. COMMIT RULE (a disclosed non-raw, item-blind output transform, in the same way as the DOPEST 0.801 floor): for yes/no questions, a temperature-scaled P(yes) strictly between 0.20 and 0.80 is reported at the nearer band edge, 0.80 if it is above 0.5, otherwise 0.20 (exactly 0.5 goes to 0.20). The argmax never changes; the Calibration axis includes the effect of this transform, and we did not measure a variant without it. LABEL-SHAPE RULES ON BY DEFAULT: hide_index_keys, describe_options and desnake_labels, which act on the shape of the labels of any request, never on item text, ids, gold labels or item lists (source review confirmed item-blind); the answer keys are always the request's own keys. The frozen offline layout counter recorded 746 describe_options, 375 noul_as_letters and 4 hide_index_keys applications across the 1,500 draw rows. TRAINING DATA (author statement, not independently verified): the model card says the training data is not disclosed, that no Decision Index or JevBench evaluation items were used in training or checkpoint selection, that the data was decontaminated against both boards' public test sets, and that official training splits of some public benchmarks were used under their licences. We did not inspect the training examples or check overlap with JevBench items; contamination of the sealed draw is unchecked. LICENCE NOTE: Apache-2.0 per the model card, LICENSE and NOTICE on the weights and per the author server licence; the base is Gemma 4 under its own terms. ANSWERED: 1,500 of 1,500, HTTP 200 on every row, no refusal, no error; none over 10 s (maximum 6.928 s; measured p50 0.052 s, p95 0.096 s). USAGE: cost is billed on the engine-processed input (chat template and system turn included), 844,031 tokens by an exact offline per-item recount; the server-native input usage, 727,045 tokens without template and system turn, equals our first-party count exactly. Output tokens are native, 9,121 in total (1 per readout plus the 7,621 reasoning tokens), priced as reported. The think pass's second read is not priced (see cost_basis). DETERMINISM (record-only): the first 50 requests were run twice; 1 of 50 was bit-identical, 0 argmax flips, maximum probability difference 0.0205 (serial vLLM path; cause not isolated; a pre-sealed smoke probe also varied between 0.80 and 0.845). The scored rows are the first run and stand. The scored loop of 1,500 requests took 260 s. ATTEMPT HISTORY: one attempt on this draw's sealed input, one create call that produced the only instance; the pre-sealed smoke, context/cut probes, think gate and TTL fit gate passed before any sealed item. HARDWARE: one Verda RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, EU); latency reflects that card.",
   "display": "Decisio 12B tuned commit-think (variant of the Decisio family)",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-27",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.14",
    "results_sha256": "9e460d48571eeb00d94f554a985fdc2ebd52b63beb9aed7a4efe88762eb7b32e",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB, driver 580.178.04), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, image 24.04.cuda13.0, USD 2.608 per hour recorded by the runner, 150 GB disk. One VM for this row, one attempt. Measured 27 Oct 2026, VM window about 17:35-17:55Z, about USD 0.8 GPU rental (elapsed time at the recorded hourly rate). Author server at 127.0.0.1:8000, network cut before sealed input (offline proof and cut proof receipts), read-only model and input, private output, HF offline flags, no credentials. Before sealed input: the 20 model files re-hashed against SHA256SUMS on the node and in the run container; the dependency Triton gate, a pre-warm that asserted the full served configuration on the scored process, and the smoke passed; the TTL fit gate admitted the run. Input sha256 9b2f32edaccb29afd726af8cd54cf1a22d85e8b1824566924045d83c6f828426 (v1.6-regular-20261027) was re-checked inside the container (match). Complete 1,500-row run, rc 0. Raw output sha256 d7396dd680e13af9ba26c4526007fee3647d03eee7673bf88075ac510f39cc3c (the pull-hash receipt). Node-shred and local-shred receipts verified; exact-id teardown receipt confirms the VM closed. GPU rental is separate from the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 68.8595406843145,
    "I_sealed": 68.38337646721313,
    "base": 68.62145857576381,
    "excess": -2.0972234253368733,
    "gap": 0.47616421710137047,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 77.2871690842039,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 71.68921044353571,
       "judge": 68.35860250494397,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 76.89855072463769,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 80,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 52.392902244101926,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.31823197645644,
       "hard": 47.96867741102309,
       "judge": 53.1200268951869,
       "standard": 57.18800006745486
      }
     },
     "sealed|choice": {
      "cc": 73.43291464351205,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 96.06943989518508,
       "hard": 55.78914008028734,
       "judge": 82.39861949956861,
       "standard": 83.95364386004013
      }
     },
     "sealed|noul": {
      "cc": 74.07916234735724,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 88.50574712643677,
       "hard": 58.53658536585367,
       "judge": 86.66666666666667,
       "standard": 79.06976744186048
      }
     },
     "sealed|score": {
      "cc": 57.6380524107701,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 84.83385160661061,
       "hard": 40.55352508302575,
       "judge": 65.91716600007275,
       "standard": 65.79053708438454
      }
     }
    }
   },
   "jevbench_score": 78.39492026890517,
   "key": "decisio-gemma-4-12b-tuned-commit-think",
   "last_measured_on": "2026-10-27",
   "licence": "Apache-2.0 per the model card, LICENSE and NOTICE on the weights and the author server; base google/gemma-4-12B-it public, ungated, under its own terms",
   "listing": "unranked",
   "measured_in": "v1.7.14",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "tachara-ai/Decisio-12b-tuned-commit-think@e36aeb243dea3008c593f454112840a1a29e5837",
   "new_in": "v1.7.14",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale: its first anchor runs used models weaker than this row, and stronger anchors are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.888,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/tachara-ai/Decisio-12b-tuned-commit-think",
   "scores": {
    "A": 78.39492026890517,
    "B": 76.22367760333563,
    "C": 78.39492026890517
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2537038250000251,
    "p50_s_raw": 0.05185191250001253,
    "p95_s_adjusted": 0.3428924033999692,
    "p95_s_raw": 0.0964462016999846
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 100
      },
      "calibration_gen": {
       "accuracy": 0.6428571428571429,
       "cc": 51.06256687371592,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": 100,
        "score": 31.109778543225698
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 51.06256687371592
      },
      "composite_gen": {
       "accuracy": 0.8947368421052632,
       "cc": 74.56843102522191,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 63.70529307566572
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 74.56843102522191
      },
      "fairness": {
       "accuracy": 1,
       "cc": 97.0395009208051,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 91.1185027624153
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 97.0395009208051
      },
      "jaggedness": {
       "accuracy": 0.6363636363636364,
       "cc": 63.2533230512745,
       "cc_by_type": {
        "choice": 62.96296296296297,
        "noul": 100,
        "score": 26.79700619086053
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 63.2533230512745
      },
      "llm_router": {
       "accuracy": 0.8222222222222222,
       "cc": 77.95486610967801,
       "cc_by_type": {
        "choice": 78.84841363102231,
        "noul": 100,
        "score": 55.01618469801169
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 77.95486610967801
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 100
      },
      "multilingual": {
       "accuracy": 0.8241042345276873,
       "cc": 72.13450652629092,
       "cc_by_type": {
        "choice": 72.51562836818282,
        "noul": 78.68852459016394,
        "score": 65.19936662052599
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 72.13450652629092
      },
      "negation_pair_gen": {
       "accuracy": 0.7272727272727273,
       "cc": 53.333333333333336,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 53.333333333333336
      },
      "option_order": {
       "accuracy": 0.8518518518518519,
       "cc": 80.95238095238095,
       "cc_by_type": {
        "choice": 80.95238095238095
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 80.95238095238095
      },
      "perturb_pair": {
       "accuracy": 0.9642857142857143,
       "cc": 91.629170201303,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 74.887510603909
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 91.629170201303
      },
      "usecase": {
       "accuracy": 0.7692307692307693,
       "cc": 64.62517349506776,
       "cc_by_type": {
        "choice": 69.36542669584243,
        "noul": 69.23076923076923,
        "score": 55.279324558591625
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 64.62517349506776
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.9523809523809523,
       "cc": 90.86616717842901,
       "cc_by_type": {
        "choice": 86.3013698630137,
        "noul": 100,
        "score": 86.29713167227331
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 90.86616717842901
      },
      "cs": {
       "accuracy": 0.7837837837837838,
       "cc": 64.71898133886172,
       "cc_by_type": {
        "choice": 59.89304812834225,
        "noul": 77.77777777777777,
        "score": 56.486118110465156
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 64.71898133886172
      },
      "da": {
       "accuracy": 0.8055555555555556,
       "cc": 57.336932107878454,
       "cc_by_type": {
        "choice": 85.09316770186335,
        "noul": 63.63636363636365,
        "score": 23.281264985408356
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 57.336932107878454
      },
      "de": {
       "accuracy": 0.8333333333333334,
       "cc": 74.08049088883853,
       "cc_by_type": {
        "choice": 81.47671726267045,
        "noul": 75,
        "score": 65.7647554038451
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 74.08049088883853
      },
      "el": {
       "accuracy": 0.6785714285714286,
       "cc": 52.12723440858542,
       "cc_by_type": {
        "choice": 42.307692307692314,
        "noul": 60.00000000000001,
        "score": 54.07401091806396
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 52.12723440858542
      },
      "en": {
       "accuracy": 0.790990990990991,
       "cc": 67.58021407116206,
       "cc_by_type": {
        "choice": 72.92780748663101,
        "noul": 73.77049180327869,
        "score": 56.042342923576484
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 67.58021407116206
      },
      "es": {
       "accuracy": 0.7567567567567568,
       "cc": 66.5867472653535,
       "cc_by_type": {
        "choice": 78.34803528468322,
        "noul": 77.77777777777777,
        "score": 43.634428733599485
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 66.5867472653535
      },
      "fi": {
       "accuracy": 0.8269230769230769,
       "cc": 72.0752146337391,
       "cc_by_type": {
        "choice": 58.85714285714286,
        "noul": 89.4736842105263,
        "score": 67.8948168335481
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 72.0752146337391
      },
      "fr": {
       "accuracy": 0.8125,
       "cc": 80.70198773146177,
       "cc_by_type": {
        "choice": 89.79591836734694,
        "noul": 100,
        "score": 52.31004482703836
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 80.70198773146177
      },
      "hi": {
       "accuracy": 0.7837837837837838,
       "cc": 63.384636065741006,
       "cc_by_type": {
        "choice": 75.20661157024793,
        "noul": 33.33333333333333,
        "score": 81.61396329364179
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 63.384636065741006
      },
      "id": {
       "accuracy": 0.9024390243902439,
       "cc": 78.39279207344903,
       "cc_by_type": {
        "choice": 81.5668202764977,
        "noul": 83.33333333333333,
        "score": 70.27822261051608
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 78.39279207344903
      },
      "it": {
       "accuracy": 0.8888888888888888,
       "cc": 76.23260462342826,
       "cc_by_type": {
        "choice": 92.01419698314109,
        "noul": 66.66666666666667,
        "score": 70.01695022047704
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 76.23260462342826
      },
      "ja": {
       "accuracy": 0.8181818181818182,
       "cc": 71.420247436298,
       "cc_by_type": {
        "choice": 61.139896373057,
        "noul": 81.81818181818181,
        "score": 71.3026641176552
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 71.420247436298
      },
      "ko": {
       "accuracy": 0.8372093023255814,
       "cc": 73.56995377668792,
       "cc_by_type": {
        "choice": 77.94117647058825,
        "noul": 80,
        "score": 62.768684859475535
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 73.56995377668792
      },
      "mixed": {
       "accuracy": 0.921875,
       "cc": 82.73368331314165,
       "cc_by_type": {
        "choice": 78.3978397839784,
        "noul": 89.99999999999999,
        "score": 79.80321015544658
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 82.73368331314165
      },
      "nl": {
       "accuracy": 0.8611111111111112,
       "cc": 59.71602506389545,
       "cc_by_type": {
        "choice": 85.84905660377359,
        "noul": 80,
        "score": 13.29901858791277
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 59.71602506389545
      },
      "no": {
       "accuracy": 0.803921568627451,
       "cc": 63.101866739084926,
       "cc_by_type": {
        "choice": 74.18244406196212,
        "noul": 73.33333333333334,
        "score": 41.78982282195933
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 63.101866739084926
      },
      "pl": {
       "accuracy": 0.7543859649122807,
       "cc": 62.72646610643087,
       "cc_by_type": {
        "choice": 61.681745609366686,
        "noul": 76.47058823529412,
        "score": 50.027064474631814
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 62.72646610643087
      },
      "pt": {
       "accuracy": 0.7872340425531915,
       "cc": 67.01723198615325,
       "cc_by_type": {
        "choice": 76.03993344425956,
        "noul": 66.66666666666667,
        "score": 58.345095847533514
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 67.01723198615325
      },
      "sv": {
       "accuracy": 0.8260869565217391,
       "cc": 73.23784832941068,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 83.33333333333333,
        "score": 69.89976472752444
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 73.23784832941068
      },
      "tr": {
       "accuracy": 0.782608695652174,
       "cc": 78.44430778057668,
       "cc_by_type": {
        "choice": 58.950969213226905,
        "noul": 100,
        "score": 76.38195412850312
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 78.44430778057668
      },
      "uk": {
       "accuracy": 0.8035714285714286,
       "cc": 71.20267319427813,
       "cc_by_type": {
        "choice": 61.64383561643836,
        "noul": 85.71428571428572,
        "score": 66.24989825211033
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 71.20267319427813
      },
      "zh": {
       "accuracy": 0.9024390243902439,
       "cc": 79.91751059303793,
       "cc_by_type": {
        "choice": 75.20661157024794,
        "noul": 100,
        "score": 64.54592020886581
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 79.91751059303793
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8013333333333333,
       "cc": 72.90115286295983,
       "cc_by_type": {
        "choice": 72.90115286295983
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 72.90115286295983
      },
      "noul": {
       "accuracy": 0.888,
       "cc": 77.60000000000001,
       "cc_by_type": {
        "noul": 77.60000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 77.60000000000001
      },
      "score": {
       "accuracy": 0.7573333333333333,
       "cc": 61.80102047223768,
       "cc_by_type": {
        "score": 61.80102047223768
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 61.80102047223768
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.625,
       "cc": 63.16991107900558,
       "cc_by_type": {
        "choice": 33.283914010378055,
        "noul": 100,
        "score": 56.2258192266387
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 63.16991107900558
      },
      "code_linting": {
       "accuracy": 0.855072463768116,
       "cc": 71.74044574412825,
       "cc_by_type": {
        "choice": 96.32578077158605,
        "noul": 66.66666666666667,
        "score": 52.22888979413203
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 71.74044574412825
      },
      "customer_support": {
       "accuracy": 0.9090909090909091,
       "cc": 78.16193533030763,
       "cc_by_type": {
        "choice": 88.92988929889299,
        "noul": 60.00000000000001,
        "score": 85.55591669202988
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 78.16193533030763
      },
      "demand_forecasting": {
       "accuracy": 0.6818181818181818,
       "cc": 43.338857767154956,
       "cc_by_type": {
        "choice": 59.05989385898408,
        "noul": 38.46153846153846,
        "score": 32.495140980942324
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 43.338857767154956
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8064516129032258,
       "cc": 67.79914758380244,
       "cc_by_type": {
        "choice": 61.99859254046446,
        "noul": 64.70588235294117,
        "score": 76.69296785800167
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 67.79914758380244
      },
      "feature_extraction": {
       "accuracy": 0.9027777777777778,
       "cc": 80.5355116886491,
       "cc_by_type": {
        "choice": 81.15577889447238,
        "noul": 85.71428571428572,
        "score": 74.73647045718921
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 80.5355116886491
      },
      "financial_crime": {
       "accuracy": 0.7777777777777778,
       "cc": 51.767921330974495,
       "cc_by_type": {
        "choice": 74.48165869218504,
        "noul": 45.45454545454546,
        "score": 35.36755984619299
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 51.767921330974495
      },
      "gaming": {
       "accuracy": 0.6206896551724138,
       "cc": 60.272336898560724,
       "cc_by_type": {
        "choice": 52.19123505976096,
        "noul": 77.77777777777777,
        "score": 50.847997858143444
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 60.272336898560724
      },
      "insurance_claims": {
       "accuracy": 0.7096774193548387,
       "cc": 49.47768217561781,
       "cc_by_type": {
        "choice": 46.54788418708242,
        "noul": 64.70588235294117,
        "score": 37.17927998682982
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 49.47768217561781
      },
      "knowledge_graphs": {
       "accuracy": 0.7910447761194029,
       "cc": 65.22938806779337,
       "cc_by_type": {
        "choice": 64.01799100449774,
        "noul": 84.61538461538463,
        "score": 47.05478858349773
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 65.22938806779337
      },
      "lead_generation": {
       "accuracy": 0.8933333333333333,
       "cc": 78.96895232119851,
       "cc_by_type": {
        "choice": 83.07475317348378,
        "noul": 82.85714285714285,
        "score": 70.97496093296891
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 78.96895232119851
      },
      "legal_compliance": {
       "accuracy": 0.7857142857142857,
       "cc": 67.52114589780652,
       "cc_by_type": {
        "choice": 63.89167502507522,
        "noul": 71.42857142857142,
        "score": 67.24319123977291
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 67.52114589780652
      },
      "llm_guardrails": {
       "accuracy": 0.8732394366197183,
       "cc": 79.20894986771259,
       "cc_by_type": {
        "choice": 87.78004073319754,
        "noul": 92.3076923076923,
        "score": 57.53911656224794
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 79.20894986771259
      },
      "model_routing": {
       "accuracy": 0.8445945945945946,
       "cc": 76.4969918746268,
       "cc_by_type": {
        "choice": 85.97888177254633,
        "noul": 88.23529411764706,
        "score": 55.276799733687
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 76.4969918746268
      },
      "moderation_trust_safety": {
       "accuracy": 0.8813559322033898,
       "cc": 78.56121114713254,
       "cc_by_type": {
        "choice": 73.8562091503268,
        "noul": 88.23529411764706,
        "score": 73.59213017342374
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 78.56121114713254
      },
      "other": {
       "accuracy": 0.71875,
       "cc": 62.04223178315643,
       "cc_by_type": {
        "choice": 60.04439511653718,
        "noul": 100,
        "score": 26.08230023293212
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 62.04223178315643
      },
      "recruiting": {
       "accuracy": 0.8461538461538461,
       "cc": 70.78638768694869,
       "cc_by_type": {
        "choice": 74.04470079307859,
        "noul": 92,
        "score": 46.31446226776745
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 70.78638768694869
      },
      "risk_assessment": {
       "accuracy": 0.8125,
       "cc": 69.50315619130384,
       "cc_by_type": {
        "choice": 62.34309623430962,
        "noul": 84.61538461538463,
        "score": 61.5509877242173
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 69.50315619130384
      },
      "scientific_discovery": {
       "accuracy": 0.8095238095238095,
       "cc": 64.67998586401727,
       "cc_by_type": {
        "choice": 70.55214723926382,
        "noul": 60.00000000000001,
        "score": 63.487810352787974
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 64.67998586401727
      },
      "search_retrieval": {
       "accuracy": 0.9402985074626866,
       "cc": 90.81068140316665,
       "cc_by_type": {
        "choice": 100,
        "noul": 85.71428571428572,
        "score": 86.71775849521424
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 90.81068140316665
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.9937575569724,
      89.99759026523022
     ],
     "I": [
      64.63338331623864,
      72.08459310262894
     ],
     "capability": [
      75.32409979820464,
      80.15708037200338
     ],
     "composite": null,
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.68433331756145,
     "capability": 78.65289594666262,
     "intelligence": 68.62145857576381
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8013333333333333,
      "calibration": 87.68445744335234,
      "cc": 72.90115286295983,
      "cc_by_type": {
       "choice": 72.90115286295983
      },
      "cc_tiered": 75.36004186385797,
      "cc_tiered_open": 77.2871690842039,
      "cc_tiered_sealed": 73.43291464351205,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 72.90115286295983
     },
     "noul": {
      "accuracy": 0.888,
      "calibration": 91.58875747975821,
      "cc": 77.60000000000001,
      "cc_by_type": {
       "noul": 77.60000000000001
      },
      "cc_tiered": 75.48885653599746,
      "cc_tiered_open": 76.89855072463769,
      "cc_tiered_sealed": 74.07916234735724,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 77.60000000000001
     },
     "score": {
      "accuracy": 0.7573333333333333,
      "calibration": 86.77978502957379,
      "cc": 61.80102047223768,
      "cc_by_type": {
       "score": 61.80102047223768
      },
      "cc_tiered": 55.01547732743602,
      "cc_tiered_open": 52.392902244101926,
      "cc_tiered_sealed": 57.6380524107701,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 61.80102047223768
     }
    },
    "raw_on_A": {
     "calibration": 84.91206086418038,
     "capability": 76.9719522949323,
     "intelligence": 69.03184372568421
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 75.47594121389008,
    "Emphasis on Accuracy 60:20:20": 72.57613764932263,
    "Emphasis on Cost 20:20:60": 73.50336799776177,
    "Emphasis on Speed 20:60:20": 80.87792602746622,
    "Intelligence only": 68.62145857576381
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Amin Roudaki (@aminry)",
   "axes": {
    "calibration": 89.15794375860679,
    "cost": 67.72536476642883,
    "intelligence": 52.324988528776075,
    "speed": 86.316096866496
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07110019971983926,
      "ece_score": 85.77996005603214,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 85.77996005603214,
      "score_before_time_limit": 85.77996005603214,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.18328840970350402,
      "brier": 0.10602428963864989,
      "ece": 0.056082845451344523,
      "n": 371,
      "score": 88.7834309097311,
      "score_before_time_limit": 88.7834309097311,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.986449864498645,
      "interval90_width": 0.49186991869918684,
      "n": 369,
      "nrps": 0.073791213570342,
      "score": 92.91044031005717,
      "score_before_time_limit": 92.91044031005717,
      "time_limit_failed_share": 0,
      "top_ece": 0.03399999011425735
     }
    },
    "score": 89.15794375860679
   },
   "capability": 70.74146614369144,
   "class": "Jev-compatible decision readout on frozen base",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 779,965 input + 1,477 output tokens over 1,477 decisions = 529.07379824 tokens/decision. GPU-time cost USD 0.0119075160195099/1,000; old list-price cost USD 0.0265536899119838/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0119075160195099/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.02614653333333334
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011907516019509904
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decisio v0.8.0 on gemma-4-12B-it (frozen, one forward pass, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. github.com/aminry/decisio tag v0.8.0 (5b42101a191d222062a795194b0aedb5003bf6ea, verified against the GitHub refs API), installed with their own `uv sync --extra serve --frozen` so their lockfile fixes every version, and served with their own `python -m decisio.serve.vllm_engine --base gemma-4-12b`. --base carries google/gemma-4-12B-it@707f0a3b (bf16, 22.8 GiB) and that profile's measured settings - temperature 3.592 for every question type, a system turn, the chat template's own answer position, every single-token form of each letter summed. Frozen base: no fine-tuning, no adapter, no task registered. One RTX 6000 Ada 48 GB (Lium, Toronto). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 54, GitHub #169",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 6000 Ada",
   "intelligence": {
    "I_open": 53.076253329484096,
    "I_sealed": 51.57372372806806,
    "base": 52.324988528776075,
    "excess": -1.0708580410222077,
    "gap": 1.502529601416036,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 72.23658792816236,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 67.9144385026738,
       "judge": 60.44825313117996,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 43.54106280193237,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 39.13043478260869,
       "judge": 30.000000000000004,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 43.45110925835756,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.14556472904645,
       "hard": 43.95984283175309,
       "judge": 46.337851474151414,
       "standard": 33.75630105253128
      }
     },
     "sealed|choice": {
      "cc": 65.8737458551212,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 85.3669351858695,
       "hard": 57.74886877828053,
       "judge": 67.41154562383613,
       "standard": 70.07020569035596
      }
     },
     "sealed|noul": {
      "cc": 40.79862969333631,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 8.737864077669899,
       "judge": 47.36842105263157,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 48.04879563574668,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 67.31002438791882,
       "hard": 37.90772388734433,
       "judge": 56.21653541756339,
       "standard": 46.44871508374021
      }
     }
    }
   },
   "jevbench_score": 70.5738473018771,
   "key": "decisio-gemma-4-12b-v080",
   "last_measured_on": "2026-10-06",
   "licence": "Apache-2.0 software; Gemma model terms",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "aminry/decisio@5b42101a191d222062a795194b0aedb5003bf6ea + google/gemma-4-12B-it@707f0a3b",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9075907590759076,
    "decisive_rate": 0.816711590296496,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.15794375860679,
     "cost": 59.34583145470655,
     "intelligence": 52.324988528776075,
     "speed": 86.9860428376769
    },
    "board_release": "v1.6.43",
    "capability": 70.74146614369144,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 24,
    "capability_rank": 42,
    "composite_ci95": {
     "A": [
      57.374888808052155,
      69.64960281748938
     ],
     "B": [
      53.18263176533343,
      66.57834556173977
     ],
     "C": [
      39.84367278336956,
      61.696012278542455
     ]
    },
    "jevbench_score": 68.17346094077045,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 17,
    "rank": 36,
    "ranks": {
     "A": 36,
     "B": 45,
     "C": 48,
     "capability": 72
    },
    "scores": {
     "A": 68.17346094077045,
     "B": 64.2795916317687,
     "C": 51.847894540473604
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3548810470150784,
     "p50_s_raw": 0.10244052350753918,
     "p95_s_adjusted": 0.564043904747814,
     "p95_s_raw": 0.20702195237390697
    },
    "views": {
     "Balanced 33:33:33": 63.21404479691977,
     "Emphasis on Accuracy 60:20:20": 58.35636046255001,
     "Emphasis on Cost 20:20:60": 61.60778501333468,
     "Emphasis on Speed 20:60:20": 70.97231847224265,
     "Intelligence only": 52.324988528776075
    }
   },
   "provenance": {
    "carried_metadata_from": "r54/r55/r61 registry and pinned author package",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 70,
   "ranked": true,
   "ranks": {
    "A": 70,
    "B": 86,
    "C": 94,
    "capability": 92
   },
   "repo": "https://github.com/aminry/decisio",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.5738473018771,
    "B": 65.97215837649057,
    "C": 53.67345799566001
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3814489560085349,
    "p50_s_raw": 0.11572447800426744,
    "p95_s_adjusted": 0.6122853703564033,
    "p95_s_raw": 0.23114268517820163
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.58456964369535,
      89.43375634811495
     ],
     "I": [
      46.8581794615679,
      56.47360933655171
     ],
     "capability": [
      66.07776568391779,
      72.37388803479945
     ],
     "composite": null,
     "seed": 39
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.15794375860679,
     "capability": 70.74146614369144,
     "intelligence": 52.324988528776075
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7493333333333333,
      "calibration": 85.77996005603214,
      "cc": 67.9433134715762,
      "cc_by_type": {
       "choice": 67.9433134715762
      },
      "cc_tiered": 69.05516689164179,
      "cc_tiered_open": 72.23658792816236,
      "cc_tiered_sealed": 65.8737458551212,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.9433134715762
     },
     "noul": {
      "accuracy": 0.7333333333333333,
      "calibration": 88.7834309097311,
      "cc": 46.66666666666666,
      "cc_by_type": {
       "noul": 46.66666666666666
      },
      "cc_tiered": 42.16984624763434,
      "cc_tiered_open": 43.54106280193237,
      "cc_tiered_sealed": 40.79862969333631,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 46.66666666666666
     },
     "score": {
      "accuracy": 0.616,
      "calibration": 92.91044031005717,
      "cc": 47.02582727916151,
      "cc_by_type": {
       "score": 47.02582727916151
      },
      "cc_tiered": 45.749952447052124,
      "cc_tiered_open": 43.45110925835756,
      "cc_tiered_sealed": 48.04879563574668,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 47.02582727916151
     }
    },
    "raw_on_A": {
     "calibration": 85.4557717128137,
     "capability": 69.50074934822587,
     "intelligence": 53.54572698363802
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.98893264498693,
    "Emphasis on Accuracy 60:20:20": 59.747989545623426,
    "Emphasis on Cost 20:20:60": 66.67270975115538,
    "Emphasis on Speed 20:60:20": 72.85144542633194,
    "Intelligence only": 52.324988528776075
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Amin Roudaky / @aminry (roudaky@gmail.com)",
   "axes": {
    "calibration": 87.60965425617285,
    "cost": 67.72536476642883,
    "intelligence": 52.299106200484864,
    "speed": 85.2750295056469
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08060951899685599,
      "ece_score": 83.8780962006288,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 83.8780962006288,
      "score_before_time_limit": 83.8780962006288,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.18867924528301883,
      "brier": 0.10747754831036967,
      "ece": 0.06459999777321335,
      "n": 371,
      "score": 87.08000044535733,
      "score_before_time_limit": 87.08000044535733,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.991869918699187,
      "interval90_width": 0.4905149051490515,
      "n": 369,
      "nrps": 0.07376404038589723,
      "score": 91.87086612253242,
      "score_before_time_limit": 91.87086612253242,
      "time_limit_failed_share": 0,
      "top_ece": 0.044409318581727114
     }
    },
    "score": 87.60965425617285
   },
   "capability": 69.95438022832886,
   "class": "Jev-compatible decision readout on frozen base",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 779,965 input + 1,477 output tokens over 1,477 decisions = 529.07379824 tokens/decision. GPU-time cost USD 0.0119075160195099/1,000; old list-price cost USD 0.0265536899119838/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0119075160195099/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.02614653333333334
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011907516019509904
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decisio v0.9.0 on gemma-4-12B-it (frozen, one prefill per question, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 21:11:36-21:15:32Z on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), 1,477 of 1,500 answered. github.com/aminry/decisio tag v0.9.0 = 333a0a849de7658969a8f8933b34a8097a8c3489 (verified against the GitHub refs API before the run and re-read as the pod's checked-out HEAD after it), Apache-2.0, installed with their own `uv sync --extra serve --frozen` so their lockfile fixes every version (decisio 0.9.0, torch 2.13.0+cu130, vllm 0.30.0, transformers 5.18.0) and served with their own command `python -m decisio.serve.vllm_engine --base gemma-4-12b`. --base carries google/gemma-4-12B-it@707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7 (bf16, 22.8 GiB) and that profile's own settings - temperature 3.592 for every question type, a system turn, the spaced layout read at the chat template's own answer position, every single-token form of each letter summed. Frozen base: no fine-tuning, no adapter, no task registered. UNCHANGED typesafe adapter and UNCHANGED run_v16.py; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. A FOLLOW-UP OF THE EARLIER decisio-gemma-4-12b-v080 ROW (run 54, same pool, same basis, same rate), measured as a CANDIDATE: the two serving versions are reported separately. WHAT v0.9.0 CHANGES is prefix-cache state-boundary registration, request ticketing and engine-health probing; the whole readout package, serve/systemone.py, serve/temperature.py, serve/abstention.py and the entire vllm_plugin are BYTE-IDENTICAL between the two tags, and no temperature or prompt line moved (r61/receipts/DECISIO-V090-SOURCE-DIFF.json). THE 23 LONG ITEMS FAIL WITH A BARE HTTP 500 from the author's own server - vLLM raises VLLMValidationError because the rendered prompt exceeds the 32,768-token context the profile declares, and the server does not translate that into the 422 the harness scores as a refusal. They are counted wrong either way and they are exactly the 23 items the v1.6.1 common cost basis excludes (the ones the Jev reference itself refuses), but it is the author's own code and A FINDING TO REPORT, NOT A THING TO PATCH AROUND; no setting of theirs was changed and no retry was made. One NVIDIA RTX 6000 Ada Generation 48 GB (Lium, Toronto, CANADA) - the same card model and city on which run 54 measured the v0.8.0 row, chosen over the 96 GB class the author names because the 12B profile is proven to fit it. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline after setup, credential-free. add-requests run 61, PREREG-R61-AMENDMENT-2, benchmarkheaven.com/submit ref 65B71BD3 (bh-submit f05c3298-702f-4314-bc8e-f823d9bce858); previously GitHub #169 for v0.8.0",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 6000 Ada Generation",
   "intelligence": {
    "I_open": 53.066754472907135,
    "I_sealed": 51.53145792806259,
    "base": 52.299106200484864,
    "excess": -1.038091097593702,
    "gap": 1.5352965448445417,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 72.23658792816236,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 67.9144385026738,
       "judge": 60.44825313117996,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 43.54106280193237,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 39.13043478260869,
       "judge": 30.000000000000004,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 43.422612688626685,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.00297563421439,
       "hard": 44.200521547084826,
       "judge": 45.87070740823223,
       "standard": 33.90447141950822
      }
     },
     "sealed|choice": {
      "cc": 65.83450996318003,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 84.03665656640308,
       "hard": 56.730769230769226,
       "judge": 68.34264432029796,
       "standard": 71.17871659071314
      }
     },
     "sealed|noul": {
      "cc": 40.79862969333631,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 8.737864077669899,
       "judge": 47.36842105263157,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 47.96123412767146,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.87191488571192,
       "hard": 38.251545396524136,
       "judge": 55.512516311610604,
       "standard": 46.59834793503713
      }
     }
    }
   },
   "jevbench_score": 70.14180529430709,
   "key": "decisio-gemma-4-12b-v090",
   "last_measured_on": "2026-10-07",
   "licence": "Apache-2.0 software; Gemma model terms",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "aminry/decisio@333a0a849de7658969a8f8933b34a8097a8c3489 + google/gemma-4-12B-it@707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9136212624584718,
    "decisive_rate": 0.8113207547169812,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.60965425617285,
     "cost": 59.34583145470655,
     "intelligence": 52.299106200484864,
     "speed": 85.86778537116794
    },
    "board_release": "v1.6.43",
    "capability": 69.95438022832886,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 27,
    "capability_rank": 47,
    "composite_ci95": {
     "A": [
      54.37922382108821,
      69.5635857308408
     ],
     "B": [
      50.29209403611676,
      66.65944464115974
     ],
     "C": [
      37.763349875755715,
      62.76631856626907
     ]
    },
    "jevbench_score": 67.76072338665118,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 19,
    "rank": 39,
    "ranks": {
     "A": 39,
     "B": 46,
     "C": 49,
     "capability": 79
    },
    "scores": {
     "A": 67.76072338665118,
     "B": 63.97786249241387,
     "C": 51.483026133201925
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4135970369447023,
     "p50_s_raw": 0.13179851847235113,
     "p95_s_adjusted": 0.6261004906438755,
     "p95_s_raw": 0.23805024532193778
    },
    "views": {
     "Balanced 33:33:33": 63.00273578153404,
     "Emphasis on Accuracy 60:20:20": 58.2353176446816,
     "Emphasis on Cost 20:20:60": 61.48719355296207,
     "Emphasis on Speed 20:60:20": 70.51330683201019,
     "Intelligence only": 52.299106200484864
    }
   },
   "provenance": {
    "carried_metadata_from": "r54/r55/r61 registry and pinned author package",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 77,
   "ranked": true,
   "ranks": {
    "A": 77,
    "B": 90,
    "C": 95,
    "capability": 101
   },
   "repo": "https://github.com/aminry/decisio",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.14180529430709,
    "B": 65.6615051029134,
    "C": 53.29211694496402
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.43738937998423355,
    "p50_s_raw": 0.14369468999211676,
    "p95_s_adjusted": 0.6786233761755284,
    "p95_s_raw": 0.2643116880877642
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.75352521796918,
      88.88216270564548
     ],
     "I": [
      45.89453540642751,
      56.94412894091952
     ],
     "capability": [
      65.24251668071038,
      72.31935391386574
     ],
     "composite": null,
     "seed": 40
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.60965425617285,
     "capability": 69.95438022832886,
     "intelligence": 52.299106200484864
    },
    "per_type": {
     "choice": {
      "accuracy": 0.748,
      "calibration": 83.8780962006288,
      "cc": 67.7727991815314,
      "cc_by_type": {
       "choice": 67.7727991815314
      },
      "cc_tiered": 69.0355489456712,
      "cc_tiered_open": 72.23658792816236,
      "cc_tiered_sealed": 65.83450996318003,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.7727991815314
     },
     "noul": {
      "accuracy": 0.7333333333333333,
      "calibration": 87.08000044535733,
      "cc": 46.66666666666666,
      "cc_by_type": {
       "noul": 46.66666666666666
      },
      "cc_tiered": 42.16984624763434,
      "cc_tiered_open": 43.54106280193237,
      "cc_tiered_sealed": 40.79862969333631,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 46.66666666666666
     },
     "score": {
      "accuracy": 0.6133333333333333,
      "calibration": 91.87086612253242,
      "cc": 47.01542304941684,
      "cc_by_type": {
       "score": 47.01542304941684
      },
      "cc_tiered": 45.69192340814907,
      "cc_tiered_open": 43.422612688626685,
      "cc_tiered_sealed": 47.96123412767146,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 47.01542304941684
     }
    },
    "raw_on_A": {
     "calibration": 83.89503312669699,
     "capability": 68.67777484474954,
     "intelligence": 53.460516562802084
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.77063013455505,
    "Emphasis on Accuracy 60:20:20": 59.62699552446956,
    "Emphasis on Cost 20:20:60": 66.538825310153,
    "Emphasis on Speed 20:60:20": 72.39390363806349,
    "Intelligence only": 52.299106200484864
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Amin Roudaki (@aminry)",
   "axes": {
    "calibration": 88.71627256952715,
    "cost": 54.98929904529576,
    "intelligence": 70.42321091828492,
    "speed": 91.13376683561437
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06150753988606317,
      "ece_score": 87.69849202278736,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 87.69849202278736,
      "score_before_time_limit": 87.69849202278736,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.06469002695417791,
      "brier": 0.04879523639868886,
      "ece": 0.04694201082318837,
      "n": 371,
      "score": 90.61159783536232,
      "score_before_time_limit": 90.61159783536232,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9349593495934959,
      "interval90_width": 0.1980578139114724,
      "n": 369,
      "nrps": 0.057879253805657324,
      "score": 87.83872785043177,
      "score_before_time_limit": 87.83872785043177,
      "time_limit_failed_share": 0,
      "top_ece": 0.09267309459285371
     }
    },
    "score": 88.71627256952715
   },
   "capability": 79.56974174390604,
   "class": "Jev-compatible decision readout on frozen base",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 779,965 input + 1,477 output tokens over 1,477 decisions = 529.07379824 tokens/decision. GPU-time cost USD 0.0316487599623216/1,000; old list-price cost USD 0.0478666418415707/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0316487599623216/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.04713268666666666
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03164875996232158
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decisio v0.8.0 on gemma-4-31B-it (frozen, FP8 on load, one prefill per state, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. github.com/aminry/decisio tag v0.8.0 (5b42101a191d222062a795194b0aedb5003bf6ea) - the same tag whose other two entries run 54 measured - installed with their own `uv sync --extra serve --frozen` so their lockfile fixes every version, and served with their own `python -m decisio.serve.vllm_engine --base gemma-4-31b`. --base carries google/gemma-4-31B-it@842da379, the official weights, FROZEN: no fine-tuning, no adapter, no task registered; vLLM 0.30.0 quantizes it to FP8 when it loads (30.6 GiB). That profile's measured settings: temperature 4.672 for choice questions and 5.252 for the rest, the 12B entry's prompt unchanged, a system turn, the chat template's own answer position, every single-token form of each letter summed, no tokens generated. New in 0.8.0 on this base and the reason it is worth its own row: several questions of a state are scored in ONE pass after a single prefill. The author's own limit is a 32,768-token prompt, so the pool's 23 items at ~80,000 tokens are a documented capacity limit, not a defect. THE AUTHOR STATES ONE 96 GB CARD; this is one H100 80 GB (Lium) and no flag of theirs was changed to make it fit. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 55, GitHub #169",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 67.59136348711468,
    "I_sealed": 73.25505834945518,
    "base": 70.42321091828492,
    "excess": -8.23708250477875,
    "gap": -5.663694862340506,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.82494232345373,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 69.80182447310474,
       "judge": 64.40342781806196,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 73.1207729468599,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 58.8283751910304,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.01492747107596,
       "hard": 63.1937957087831,
       "judge": 57.45226814139626,
       "standard": 46.068418589953396
      }
     },
     "sealed|choice": {
      "cc": 83.09170328497459,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 90.68804966373514,
       "hard": 76.07466063348416,
       "judge": 86.03351955307262,
       "standard": 88.91489099642814
      }
     },
     "sealed|noul": {
      "cc": 76.04956278293162,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 61.165048543689316,
       "judge": 81.57894736842107,
       "standard": 91.54929577464787
      }
     },
     "sealed|score": {
      "cc": 60.62390898045936,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 72.91119795662057,
       "hard": 56.16498434122372,
       "judge": 63.285909697701555,
       "standard": 59.4051126949867
      }
     }
    }
   },
   "jevbench_score": 73.21985514490987,
   "key": "decisio-gemma-4-31b-v080",
   "last_measured_on": "2026-10-06",
   "licence": "Apache-2.0 software; Gemma model terms",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "aminry/decisio@5b42101a191d222062a795194b0aedb5003bf6ea + google/gemma-4-31B-it@842da379",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9596541786743515,
    "decisive_rate": 0.9353099730458221,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.71627256952715,
     "cost": 51.66530928015878,
     "intelligence": 70.42321091828492,
     "speed": 91.18580810090454
    },
    "board_release": "v1.6.43",
    "capability": 79.56974174390604,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 7,
    "capability_rank": 14,
    "composite_ci95": {
     "A": [
      69.85754247280484,
      72.80874617920621
     ],
     "B": [
      68.87932855683242,
      73.03493454427631
     ],
     "C": [
      69.85754247280484,
      72.80874617920621
     ]
    },
    "jevbench_score": 71.6926576652985,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 5,
    "rank": 16,
    "ranks": {
     "A": 16,
     "B": 13,
     "C": 16,
     "capability": 26
    },
    "scores": {
     "A": 71.6926576652985,
     "B": 71.43512018650192,
     "C": 71.6926576652985
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24419243037700653,
     "p50_s_raw": 0.047096215188503265,
     "p95_s_adjusted": 0.3116642528679222,
     "p95_s_raw": 0.0808321264339611
    },
    "views": {
     "Balanced 33:33:33": 67.38267528356181,
     "Emphasis on Accuracy 60:20:20": 68.56682969127965,
     "Emphasis on Cost 20:20:60": 60.07267047173585,
     "Emphasis on Speed 20:60:20": 75.23880708945377,
     "Intelligence only": 70.42321091828492
    }
   },
   "provenance": {
    "carried_metadata_from": "r54/r55/r61 registry and pinned author package",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 44,
   "ranked": true,
   "ranks": {
    "A": 44,
    "B": 32,
    "C": 37,
    "capability": 37
   },
   "repo": "https://github.com/aminry/decisio",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.21985514490987,
    "B": 72.64289668203361,
    "C": 73.21985514490987
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24408713364973664,
    "p50_s_raw": 0.04704356682486832,
    "p95_s_adjusted": 0.315557443536818,
    "p95_s_raw": 0.082778721768409
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-31B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.0517072217947,
      90.77969936508954
     ],
     "I": [
      65.27867139621,
      74.3007710998681
     ],
     "capability": [
      75.67457118842692,
      81.98658905786246
     ],
     "composite": null,
     "seed": 41
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.71627256952715,
     "capability": 79.56974174390604,
     "intelligence": 70.42321091828492
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8546666666666667,
      "calibration": 87.69849202278736,
      "cc": 81.41394238511599,
      "cc_by_type": {
       "choice": 81.41394238511599
      },
      "cc_tiered": 76.95832280421416,
      "cc_tiered_open": 70.82494232345373,
      "cc_tiered_sealed": 83.09170328497459,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.41394238511599
     },
     "noul": {
      "accuracy": 0.888,
      "calibration": 90.61159783536232,
      "cc": 77.60000000000001,
      "cc_by_type": {
       "noul": 77.60000000000001
      },
      "cc_tiered": 74.58516786489577,
      "cc_tiered_open": 73.1207729468599,
      "cc_tiered_sealed": 76.04956278293162,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 77.60000000000001
     },
     "score": {
      "accuracy": 0.7093333333333334,
      "calibration": 87.83872785043177,
      "cc": 60.762544130610976,
      "cc_by_type": {
       "score": 60.762544130610976
      },
      "cc_tiered": 59.72614208574488,
      "cc_tiered_open": 58.8283751910304,
      "cc_tiered_sealed": 60.62390898045936,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 60.762544130610976
     }
    },
    "raw_on_A": {
     "calibration": 84.78496723140272,
     "capability": 77.99664929333233,
     "intelligence": 71.20833135526192
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.1912216103893,
    "Emphasis on Accuracy 60:20:20": 69.678808277715,
    "Emphasis on Cost 20:20:60": 62.71258459722742,
    "Emphasis on Speed 20:60:20": 76.56514800986751,
    "Intelligence only": 70.42321091828492
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "aminry (Decisio server); Qwen (base weights)",
   "axes": {
    "calibration": 89.00828628793406,
    "cost": 61.71356220552948,
    "intelligence": 33.9677591411048,
    "speed": 90.99460763280192
   },
   "base_model": "Qwen/Qwen3.6-35B-A3B",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06928264937590087,
      "ece_score": 86.14347012481983,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 86.14347012481983,
      "score_before_time_limit": 86.14347012481983,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.46112600536193027,
      "brier": 0.1284195924246751,
      "ece": 0.04370007337368959,
      "n": 373,
      "score": 91.25998532526208,
      "score_before_time_limit": 91.25998532526208,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9704301075268817,
      "interval90_width": 0.561379928315412,
      "n": 372,
      "nrps": 0.1005888218366877,
      "score": 89.62140341372029,
      "score_before_time_limit": 89.62140341372029,
      "time_limit_failed_share": 0,
      "top_ece": 0.05349155494445332
     }
    },
    "score": 89.00828628793406
   },
   "capability": 61.48802271451943,
   "code_revision": "THE AUTHOR'S OWN SERVER, run unmodified: aminry/decisio at 333a0a849de7658969a8f8933b34a8097a8c3489 (= tag v0.9.0), installed on the disposable rented pod with the author's own 'uv sync --extra serve --frozen' against uv.lock sha256 38e9d9383bb838602b4ad7f0d9ed6ad2408d5b5b9fc05cb2c9432e09c4a5e567. Frozen stack: decisio 0.9.0, vLLM 0.30.0, torch 2.13.0, transformers 5.18.0, tokenizers 0.23.2, triton 3.7.1 and flashinfer-python 0.6.18.post1. Entrypoint: python -m decisio.serve.vllm_engine --base qwen3.6-35b-a3b --model /model --host 127.0.0.1 --port 8000. The checkpoint was passed as the hash-verified local snapshot directory instead of a bare repository id; the host is the author's default, made explicit; every other flag is the tag's default. The native POST /v1/systemone endpoint was used with the frozen v1.6 measurement driver and TypeSafe adapter, one decision per request, serial, with no response shim or instruction conversion of ours. Profile: spaced/prefill/single, no system turn, thinking off, front padding to the 1,056-token prefix-cache block, letters-keys yes/no, label-shape defaults on, temperatures 1.370 choice and 1.506 yes/no and score, orders 1, engine in-process, sequential, max_model_len 32,768, DeepGEMM off, no registered tasks, no abstention tasks, no adapters and no commit transform. Speculative decoding/MTP is OFF: the checkpoint ships an MTP file, but the frozen server does not enable it and the pre-sealed smoke and ready gates require every logged speculative_config to be None. Weights: Qwen/Qwen3.6-35B-A3B-FP8 at 95a723d08a9490559dae23d0cff1d9466213d989, Qwen's official FP8 release of Qwen/Qwen3.6-35B-A3B, 56 pinned files, 37,493,015,668 bytes, 42 safetensors weight shards including the MTP and outside files, block-FP8 e4m3 with 128 by 128 blocks and dynamic activations. Text only: the multimodal tower is not loaded. No fine-tuning, adapter or task registration is part of this served configuration. Source review (text only, no author code imported or executed on the evaluator host): no stop condition found, no branch on item text, ids, gold labels or item lists; the disclosed label-shape rules apply to every request of that shape. Author code ran only on the disposable rented pod. The run used network-none, read-only containers with capabilities dropped, no credentials, read-only source/model/input and private output; network cut and offline model-hash proofs preceded sealed input.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,761,623 input + 1,485 output tokens over 1,485 decisions = 1187.27811448 tokens/decision. GPU-time cost USD 0.0188892447198514/1,000; old list-price cost USD 0.178941717171717/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0188892447198514/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01888924471985142
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of Decisio 0.9.0 on Qwen3.6-35B-A3B in this intake (submitted through the form). A NEW ROW of the Decisio family, with a different base from the live Gemma-based Decisio rows; it replaces none of them. 'Jev-compatible' describes the interface only. SERVED WEIGHTS: base_model Qwen/Qwen3.6-35B-A3B (served: Qwen's official FP8 release at 95a723d0). These are the official Qwen FP8 weights, not a Decisio fine-tune; the server is aminry/decisio v0.9.0, run as frozen in code_revision. SIBLING CHECK: none of the 42 weight-shard LFS hashes equals an LFS blob of the 173 live row repositories in the pre-launch snapshot (171 checked; Uniphore/uniphore-4b-decider and nemotron-deciffusion/Nemotron-Deciffusion-8B-v2 returned 401 and were unchecked). This compares bytes at repository HEAD, not each published row's revision, and is not a claim of independent training. PROMPT AND READOUT: one user turn with state, instructions and lettered options, Qwen chat template with thinking disabled, and an Answer: prefill; up to 1,055 leading newline tokens pad the state prefix to the hybrid's 1,056-token block. No system turn. Yes/no is presented as A. No and B. Yes, with any side descriptions retained, and the returned probabilities are keyed back to yes/no. Choice defaults hide index-like keys, describe options and desnake bare labels; these are item-blind label-shape rules, never answer-key changes. Score levels are lettered options. One label-logprob readout per question, max_tokens 1, no generated explanation; softmax probabilities are temperature-scaled at 1.370 for choice and 1.506 for yes/no and score, preserving argmax. No commit rule, no task-specific calibration or abstention, no ensemble and no retry. TEMPERATURE FIT (author statement): the source describes fitting on the author's private 1,400-item suite; the examples were not inspected and overlap with JevBench was not independently checked. Base-model training contamination of the sealed draw is unchecked. PREFIX CACHE: standard vLLM automatic prefix caching is on, with Mamba align mode; repeated padded state prefixes can reuse KV/GDN state across requests. Its floating-point effect is part of the served configuration; the replay also reads the cache. SPECULATIVE DECODING/MTP OFF, despite the shipped MTP weights. CONTEXT AND REFUSALS: the frozen tag has no dedicated over-limit refusal handler and no truncation. An engine prompt of 32,767 tokens is accepted; 32,768 or more triggers vLLM's context-length validation exception, which this tag leaves as bare HTTP 500 'Internal Server Error'. Exactly the 15 preregistered over-length items returned that bare 500 with the required context-error log evidence and are scored as misses, without retry or partial credit. The other 1,485 items returned HTTP 200; total 1,500 rows. Native outcome deviations are record-only under the preregistered amendment, never a reason to retry or improve a score: measured outcome deviations 0. The wrapper's remaining post-sealed stops concern infrastructure/configuration and the frozen driver's own unchanged stop rule. USAGE AND BILLING: server-reported 664,205 input and 1,485 output tokens; 0 usage mismatches against the offline count. Input usage counts state and question text without padding or template; the billed input is instead the preregistered engine-prompt transcription, 1,761,623, as detailed in cost_basis. A server-observed engine count was unavailable under the frozen configuration, so the native usage agreement is not an independent validation of the engine total. The earlier preregistration estimate near 1.13x the cap used a rough usage basis and is superseded by engine-basis billing, which was over the cost cap under the earlier list-price basis; inside both caps from v1.8.0 (GPU-time cost basis). HARDWARE AND LATENCY: one RTX PRO 6000 96 GB, so these measurements do not establish latency on any other card or hosted service. Measured loopback p95 about 0.07 s, maximum about 0.13 s, 0 requests over 10 s; scored loop 91 s. No decision receives a time-limit miss in the v1.7 share view on these timings. That view uses the common self-hosted latency estimate (2 times measured p50 plus 0.15 s), an assumption about hosted latency rather than a hosted-endpoint measurement. DETERMINISM (record-only): first 50 requests replayed twice after scoring, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. The scored run stands. PREREGISTRATION: append-only amendments before any pod froze MTP off and both input totals, changed native outcome deviations to record-only, and selected engine-input billing with an explicit offline-transcription limitation. The earlier suggestion that extra head-recompute requests should enlarge this run's engine basis was corrected: with empty tasks and debug off they do not occur on the frozen route, and no phantom tokens are billed. ATTEMPT HISTORY: create calls 1-5 failed with Verda HTTP 503 before an instance existed (absence receipts retained). Call 6 was the sole measured attempt; zero sealed items reached a model before it. LICENCE: Decisio server Apache-2.0 with NOTICE; Qwen weights Apache-2.0, ungated.",
   "display": "Decisio v0.9.0 on Qwen3.6-35B-A3B (official FP8, self-hosted)",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-26",
    "release": "v1.7.10",
    "results_sha256": "f64f94ec6de6eab37bb55dfc109b1778d81968d076fdfa355f545b3ad6364f7f"
   },
   "endpoint_condition": "Evaluator-owned disposable rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB, FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, image 24.04.cuda13.0. Measured 26 Oct 2026, approximately 21:40-22:00Z, approximately USD 0.9 GPU rental, separate from the benchmark's estimated model tariff. Author server at loopback 127.0.0.1:8000, network cut before sealed input, offline model-hash proof, no credentials, read-only source/model/input, private output. Pre-sealed smoke included the cold first request at the 32,767-token boundary (1.043543 s), accepted and refused length probes and configuration gates; the ready gate preceded the scored process's pre-warm. TTL fit admitted the run with 5,606 s remaining against 1,529 s required (compute estimate 887 s; forecast over 10 s: 0). Frozen input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b (v1.6-regular-20261012). Complete 1,500-row run; scored loop 91.306016 s. Raw output sha256 9e72102fef97c070a00f88d6cd22b3fe6de99f725d535df3e338c8d20d0a16b1 (evaluator-verified). Completion receipt confirms teardown and node/local shred verified. One measured attempt after five create calls failed before any instance existed.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 38.09573346992488,
    "I_sealed": 29.839784812284723,
    "base": 33.9677591411048,
    "excess": -1.5835781127155535,
    "gap": 8.255948657640158,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 73.91756905018784,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 58.47750865051904,
       "judge": 72.31377719182598,
       "standard": 100
      }
     },
     "open|noul": {
      "cc": 8.88198757763975,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 4.347826086956519,
       "judge": 0,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 31.487643781947057,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.98931482149057,
       "hard": 31.886160365807303,
       "judge": 42.40672703529459,
       "standard": 6.061150214433519
      }
     },
     "sealed|choice": {
      "cc": 57.03473397077521,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 82.84625158831004,
       "hard": 43.79553602405458,
       "judge": 57.68854064642509,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": -10.909090909090914,
       "judge": -32.30769230769231,
       "standard": 22.388059701492537
      }
     },
     "sealed|score": {
      "cc": 32.484620466078965,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 50.20001863418941,
       "hard": 28.161694894362466,
       "judge": 29.495227521809664,
       "standard": 36.756861941860706
      }
     }
    }
   },
   "jevbench_score": 27.201182936525417,
   "key": "decisio-qwen3.6-35b-a3b-v090",
   "last_measured_on": "2026-10-26",
   "licence": "Apache-2.0 (aminry/decisio server, LICENSE and NOTICE; Qwen official FP8 weights per model card); ungated",
   "listing": "ranked",
   "measured_in": "v1.7.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "aminry/decisio@333a0a849de7658969a8f8933b34a8097a8c3489 + weights Qwen/Qwen3.6-35B-A3B-FP8@95a723d08a9490559dae23d0cff1d9466213d989",
   "new_in": "v1.7.10",
   "noul_decisive": {
    "acc_among_decisive": 0.9502487562189055,
    "decisive": 201,
    "decisive_rate": 0.5388739946380697,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 172,
   "ranked": true,
   "ranks": {
    "A": 172,
    "B": 173,
    "C": 174,
    "capability": 158
   },
   "repo": "https://github.com/aminry/decisio",
   "scores": {
    "A": 27.201182936525417,
    "B": 23.714611519240428,
    "C": 18.8897103725871
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2775734759999523,
    "p50_s_raw": 0.06378673799997614,
    "p95_s_adjusted": 0.286524176300145,
    "p95_s_raw": 0.0682620881500725
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 91.0659892395699,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 81.53130105204305
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 91.0659892395699
      },
      "calibration_gen": {
       "accuracy": 0.45544554455445546,
       "cc": 9.482314545146929,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -12,
        "score": 13.617675342757863
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 9.482314545146929
      },
      "composite_gen": {
       "accuracy": 0.5346534653465347,
       "cc": 13.783691918388092,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -62.96296296296296,
        "score": 36.834363921379264
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 13.783691918388092
      },
      "fairness": {
       "accuracy": 0.8541666666666666,
       "cc": 86.80902007756588,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 60.427060232697585
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.80902007756588
      },
      "jaggedness": {
       "accuracy": 0.45555555555555555,
       "cc": 6.897562435616547,
       "cc_by_type": {
        "choice": 40.808469682386914,
        "noul": -45.945945945945944,
        "score": 25.830163570408672
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 6.897562435616547
      },
      "llm_router": {
       "accuracy": 0.7380191693290735,
       "cc": 58.189724436755604,
       "cc_by_type": {
        "choice": 67.50902527075812,
        "noul": 54.639175257731964,
        "score": 52.42097278177673
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 58.189724436755604
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5185185185185185,
       "cc": 8.147377769478949,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -25,
        "score": 11.297803411529639
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 8.147377769478949
      },
      "multilingual": {
       "accuracy": 0.6,
       "cc": 29.226114064724115,
       "cc_by_type": {
        "choice": 62.83012716009129,
        "noul": -11.111111111111116,
        "score": 35.95932614519217
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 29.226114064724115
      },
      "negation_pair_gen": {
       "accuracy": 0.5348837209302325,
       "cc": 4.9428320687191,
       "cc_by_type": {
        "choice": 70,
        "noul": -66.66666666666667,
        "score": 11.495162872823972
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 4.9428320687191
      },
      "option_order": {
       "accuracy": 0.7291666666666666,
       "cc": 65.37949400798935,
       "cc_by_type": {
        "choice": 65.37949400798935
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 65.37949400798935
      },
      "perturb_pair": {
       "accuracy": 0.796875,
       "cc": 70.55806785154742,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 80,
        "score": 58.70123058166931
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 70.55806785154742
      },
      "usecase": {
       "accuracy": 0.5636363636363636,
       "cc": 29.482524234485222,
       "cc_by_type": {
        "choice": 64.4607843137255,
        "noul": -8.333333333333337,
        "score": 32.3201217230635
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 29.482524234485222
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 22.042187194374236,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 0,
        "score": -14.825819369258241
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 22.042187194374236
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 19.851040386013317,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 13.623781680313375
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 19.851040386013317
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 72.83554407726825,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 75,
        "score": 59.43583577162778
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 72.83554407726825
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 35.54965057716403,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 90.26161467376397
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 35.54965057716403
      },
      "en": {
       "accuracy": 0.6020851433536055,
       "cc": 31.386953898998673,
       "cc_by_type": {
        "choice": 59.800344150059175,
        "noul": 0.7194244604316502,
        "score": 33.641093086505194
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 31.386953898998673
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 57.48438588197994,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75,
        "score": 7.241085867799535
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 57.48438588197994
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 14.772730537952448,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 76.0121806848956
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 14.772730537952448
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 37.90522812856751,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 56.068625562173104
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 37.90522812856751
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 32.04747574890118,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 25,
        "score": 21.421756855642094
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 32.04747574890118
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 54.43275984299148,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": -8.642484305787846
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 54.43275984299148
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 17.414227145361995,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 19.999999999999996,
        "score": 3.72798723433857
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 17.414227145361995
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -17.994579466848403,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": -9.539293956100758
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 1,
       "cc": 98.96790594677132,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.90371784031394
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 98.96790594677132
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -15.887792227614007,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 4.284675265209925
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 32.69600528445676,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 33.32288678015826
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 32.69600528445676
      },
      "pt": {
       "accuracy": 0.6071428571428571,
       "cc": 36.27477661528186,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 19.999999999999996,
        "score": 41.859097082144935
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 36.27477661528186
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 12.528882043685115,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -60,
        "score": 48.650475918289395
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 12.528882043685115
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": 10.919700720201966,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0,
        "score": -2.023506535046282
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 10.919700720201966
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 66.33317480172293,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 63.51565343742684
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 66.33317480172293
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 10.466189386182075,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -35.268098508120445
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 10.466189386182075
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6946666666666667,
       "cc": 60.56181710327407,
       "cc_by_type": {
        "choice": 60.56181710327407
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 60.56181710327407
      },
      "noul": {
       "accuracy": 0.5093333333333333,
       "cc": 1.866666666666661,
       "cc_by_type": {
        "noul": 1.866666666666661
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 1.866666666666661
      },
      "score": {
       "accuracy": 0.528,
       "cc": 33.540431211806485,
       "cc_by_type": {
        "score": 33.540431211806485
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 33.540431211806485
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -3.489628047599728,
       "cc_by_type": {
        "choice": 68,
        "noul": -75,
        "score": -3.468884142799178
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.6666666666666666,
       "cc": 38.34141893539218,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 55.55555555555556,
        "score": -3.2642180040374225
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 38.34141893539218
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 46.034486565932774,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": 70.62378490105033
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 46.034486565932774
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -33.52753929445049,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -18.711272854111716
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 10.488497706203288,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 22.769840944696817
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 10.488497706203288
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -35.67853517727577,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100,
        "score": -18.985291066418508
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 29.957025703769418,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -14.28571428571429,
        "score": 43.37247767153236
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 29.957025703769418
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": -9.968607965636751,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100,
        "score": 22.358984465459073
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 27.587954726331535,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 25.26386417899461
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 27.587954726331535
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 13.061590169123718,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 0,
        "score": 16.27256493991933
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 13.061590169123718
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 29.7730972059441,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 22.652624951165635
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 29.7730972059441
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -24.127536448012027,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100,
        "score": 24.391584204351012
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 26.905890816523225,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -19.999999999999996,
        "score": 23.750302008110925
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 26.905890816523225
      },
      "model_routing": {
       "accuracy": 0.7880184331797235,
       "cc": 63.71451511433988,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 72.72727272727273,
        "score": 39.45523365470797
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 63.71451511433988
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 16.641540981327683,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 72.9572525025243
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 16.641540981327683
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 47.0376494132249,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 24.492706828346943
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 47.0376494132249
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -5.704275188964723,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -100,
        "score": 27.222149802563965
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": -4.089746801736752,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -66.66666666666667,
        "score": 12.368440754210031
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 92.54821291427704,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 77.64463874283113
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 92.54821291427704
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.81639830159503,
      89.65387154575536
     ],
     "I": [
      29.228221952119895,
      38.971364285034035
     ],
     "capability": [
      56.89187236485043,
      63.308204936745376
     ],
     "composite": null,
     "seed": 28
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.00828628793406,
     "capability": 61.48802271451943,
     "intelligence": 33.9677591411048
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6946666666666667,
      "calibration": 86.14347012481983,
      "cc": 60.56181710327407,
      "cc_by_type": {
       "choice": 60.56181710327407
      },
      "cc_tiered": 65.47615151048151,
      "cc_tiered_open": 73.91756905018784,
      "cc_tiered_sealed": 57.03473397077521,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.56181710327407
     },
     "noul": {
      "accuracy": 0.5093333333333333,
      "calibration": 91.25998532526208,
      "cc": 1.866666666666661,
      "cc_by_type": {
       "noul": 1.866666666666661
      },
      "cc_tiered": 4.440993788819875,
      "cc_tiered_open": 8.88198757763975,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 1.866666666666661
     },
     "score": {
      "accuracy": 0.528,
      "calibration": 89.62140341372029,
      "cc": 33.540431211806485,
      "cc_by_type": {
       "score": 33.540431211806485
      },
      "cc_tiered": 31.98613212401301,
      "cc_tiered_open": 31.487643781947057,
      "cc_tiered_sealed": 32.484620466078965,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 33.540431211806485
     }
    },
    "raw_on_A": {
     "calibration": 84.76127264319592,
     "capability": 60.032353413092366,
     "intelligence": 35.303434182988816
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 24.44801789581458,
    "Emphasis on Accuracy 60:20:20": 19.977191518805512,
    "Emphasis on Cost 20:20:60": 25.916338423955633,
    "Emphasis on Speed 20:60:20": 29.354307747376208,
    "Intelligence only": 15.676917877632853
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "vLLM Semantic Router",
   "axes": {
    "calibration": 82.20136692745442,
    "cost": 57.00207511798864,
    "intelligence": 60.4093162408047,
    "speed": 83.34942176442772
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10003484913300223,
      "ece_score": 79.99303017339956,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 79.99303017339956,
      "score_before_time_limit": 79.99303017339956,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2668463611859838,
      "brier": 0.06657517963022924,
      "ece": 0.0795050319911621,
      "n": 371,
      "score": 84.09899360176757,
      "score_before_time_limit": 84.09899360176757,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.994579945799458,
      "interval90_width": 0.5117434507678408,
      "n": 369,
      "nrps": 0.05676847122486311,
      "score": 82.51207700719613,
      "score_before_time_limit": 82.51207700719613,
      "time_limit_failed_share": 0,
      "top_ece": 0.14649499431560725
     }
    },
    "score": 82.20136692745442
   },
   "capability": 71.30534158412956,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 779,329 input + 0 output tokens over 1,477 decisions = 527.643195667 tokens/decision. GPU-time cost USD 0.0271184062684619/1,000; old list-price cost USD 0.221610142180095/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0271184062684619/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.21821211999999998
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.027118406268461865
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision 2.0 Vega 27B (vLLM-SR, Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 63.49716698880913,
    "I_sealed": 57.321465492800264,
    "base": 60.4093162408047,
    "excess": 3.6023138535706187,
    "gap": 6.1757014960088625,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 83.11168946691484,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 77.35136835482857,
       "judge": 80.22412656558997,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 48.59696342305038,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 73.91304347826086,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 58.78284807646216,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.69543007415754,
       "hard": 61.18205862546275,
       "judge": 57.862992432344406,
       "standard": 53.407919445789865
      }
     },
     "sealed|choice": {
      "cc": 75.37523860459501,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 71.49321266968325,
       "judge": 77.6536312849162,
       "standard": 76.7212710924991
      }
     },
     "sealed|noul": {
      "cc": 40.19307217860048,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": 26.21359223300972,
       "judge": 42.10526315789473,
       "standard": 63.38028169014085
      }
     },
     "sealed|score": {
      "cc": 56.396085695205315,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 62.340074967907654,
       "hard": 48.314713299858624,
       "judge": 65.93509972749679,
       "standard": 55.2783148011103
      }
     }
    }
   },
   "jevbench_score": 68.6578700666544,
   "key": "decision-2-0-vega-27b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "vllm-sr/Decision-2.0-Vega-27B@7aec49ae11a18741706da549ab626b9052795fe7",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9889705882352942,
    "decisive_rate": 0.7331536388140162,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.20136692745442,
     "cost": 29.632311028770005,
     "intelligence": 60.4093162408047,
     "speed": 83.80610555916236
    },
    "board_release": "v1.6.43",
    "capability": 71.30534158412956,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      18.452136544131196,
      19.07499323293124
     ],
     "B": [
      18.66694813760067,
      19.664978813028327
     ],
     "C": [
      15.74438986147704,
      19.07499323293124
     ]
    },
    "jevbench_score": 18.883756282485887,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 101,
    "rank": 136,
    "ranks": {
     "A": 136,
     "B": 132,
     "C": 118,
     "capability": 67
    },
    "scores": {
     "A": 18.883756282485887,
     "B": 19.308516568607676,
     "C": 18.883756282485887
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5303170211613178,
     "p50_s_raw": 0.1901585105806589,
     "p95_s_adjusted": 0.784971479140222,
     "p95_s_raw": 0.317485739570111
    },
    "views": {
     "Balanced 33:33:33": 16.931351780927532,
     "Emphasis on Accuracy 60:20:20": 18.419751896131185,
     "Emphasis on Cost 20:20:60": 13.537269231212232,
     "Emphasis on Speed 20:60:20": 20.397174151969452,
     "Intelligence only": 21.217536521841364
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 89,
   "ranked": true,
   "ranks": {
    "A": 89,
    "B": 81,
    "C": 63,
    "capability": 85
   },
   "repo": "https://huggingface.co/vllm-sr/Decision-2.0-Vega-27B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.6578700666544,
    "B": 66.83274271891666,
    "C": 68.6578700666544
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5422534752637148,
    "p50_s_raw": 0.1961267376318574,
    "p95_s_adjusted": 0.8528162744827569,
    "p95_s_raw": 0.3514081372413784
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.19756848350097,
      83.31136430016561
     ],
     "I": [
      55.369466346348005,
      64.07264251912545
     ],
     "capability": [
      67.8132497141865,
      72.6791469378467
     ],
     "composite": null,
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.20136692745442,
     "capability": 71.30534158412956,
     "intelligence": 60.4093162408047
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8213333333333334,
      "calibration": 79.99303017339956,
      "cc": 77.1510851339958,
      "cc_by_type": {
       "choice": 77.1510851339958
      },
      "cc_tiered": 79.24346403575493,
      "cc_tiered_open": 83.11168946691484,
      "cc_tiered_sealed": 75.37523860459501,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 77.1510851339958
     },
     "noul": {
      "accuracy": 0.7173333333333334,
      "calibration": 84.09899360176757,
      "cc": 43.466666666666676,
      "cc_by_type": {
       "noul": 43.466666666666676
      },
      "cc_tiered": 44.39501780082543,
      "cc_tiered_open": 48.59696342305038,
      "cc_tiered_sealed": 40.19307217860048,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 43.466666666666676
     },
     "score": {
      "accuracy": 0.688,
      "calibration": 82.51207700719613,
      "cc": 56.084109797956,
      "cc_by_type": {
       "score": 56.084109797956
      },
      "cc_tiered": 57.58946688583374,
      "cc_tiered_open": 58.78284807646216,
      "cc_tiered_sealed": 56.396085695205315,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 56.084109797956
     }
    },
    "raw_on_A": {
     "calibration": 78.63363678590805,
     "capability": 71.50200013792696,
     "intelligence": 64.37036348994586
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.08348515539174,
    "Emphasis on Accuracy 60:20:20": 63.12962409618874,
    "Emphasis on Cost 20:20:60": 61.59070201416268,
    "Emphasis on Speed 20:60:20": 71.3368422164511,
    "Intelligence only": 60.4093162408047
   }
  },
  {
   "adapter_id": "images/decision-2b/spec.json:flymy_v15.FlyMyPackage",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "FlyMy.AI (@denti)",
   "axes": {
    "calibration": 86.77033341356835,
    "cost": 66.37275756963365,
    "intelligence": 22.875473256337628,
    "speed": 90.39467608500453
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.060081291070015405,
      "ece_score": 87.98374178599691,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 87.98374178599691,
      "score_before_time_limit": 87.98374178599691,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.6091644204851752,
      "brier": 0.15626137202130647,
      "ece": 0.06929060503851944,
      "n": 371,
      "score": 86.1418789922961,
      "score_before_time_limit": 86.1418789922961,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9539295392953929,
      "interval90_width": 0.646115627822945,
      "n": 369,
      "nrps": 0.13027713115375125,
      "score": 86.18537946241204,
      "score_before_time_limit": 86.18537946241204,
      "time_limit_failed_share": 0,
      "top_ece": 0.07300763979900397
     }
    },
    "score": 86.77033341356835
   },
   "capability": 54.82290333495299,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     6.070785321244929,
     14.301729550123063
    ],
    "B": [
     4.787007736285314,
     11.888763436425752
    ],
    "C": [
     4.215823139753423,
     9.931756632029904
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013210160098522166
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision 2B (FlyMy.AI, v59)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 24.479797267586093,
    "I_sealed": 21.271149245089163,
    "base": 22.875473256337628,
    "excess": 0.6352603800586869,
    "gap": 3.2086480224969307,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 50.388518138122464,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 45.26580685750237,
       "judge": 40.672379696769944,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -56.52173913043479,
       "judge": -19.999999999999996,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 23.050873664635812,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 20.73971356294858,
       "hard": 15.558133148961872,
       "judge": 38.430985047283826,
       "standard": 16.12176767285528
      }
     },
     "sealed|choice": {
      "cc": 45.220796675917924,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 60.0916414160077,
       "hard": 37.38687782805428,
       "judge": 42.271880819366864,
       "standard": 57.876585786426915
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -4.0000000000000036,
       "hard": -53.398058252427184,
       "judge": -23.684210526315784,
       "standard": -18.309859154929576
      }
     },
     "sealed|score": {
      "cc": 18.59265105934957,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 27.03021479008173,
       "hard": 11.986008435765372,
       "judge": 25.699675889534046,
       "standard": 16.92661719587518
      }
     }
    }
   },
   "jevbench_score": 10.289709237720572,
   "key": "decision-2b",
   "last_measured_on": "2026-10-05",
   "licence": "Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9379310344827586,
    "decisive_rate": 0.3908355795148248,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.77033341356835,
     "cost": 66.37275756963365,
     "intelligence": 22.875473256337628,
     "speed": 90.39462801231525
    },
    "board_release": "v1.6.43",
    "capability": 54.82290333495299,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 84,
    "capability_rank": 110,
    "composite_ci95": {
     "A": [
      6.070784931981477,
      14.301728447276645
     ],
     "B": [
      4.787007542130122,
      11.888762823793376
     ],
     "C": [
      4.215822869431582,
      9.93175586616434
     ]
    },
    "jevbench_score": 10.289708493744515,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 131,
    "rank": 170,
    "ranks": {
     "A": 170,
     "B": 173,
     "C": 170,
     "capability": 165
    },
    "scores": {
     "A": 10.289708493744515,
     "B": 8.367001374285444,
     "C": 7.14563089843369
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.30028566285036506,
     "p50_s_raw": 0.07514283142518252,
     "p95_s_adjusted": 0.30409032410243525,
     "p95_s_raw": 0.07704516205121763
    },
    "views": {
     "Balanced 33:33:33": 8.990679348778704,
     "Emphasis on Accuracy 60:20:20": 6.6544725899582415,
     "Emphasis on Cost 20:20:60": 10.468172682915556,
     "Emphasis on Speed 20:60:20": 11.379622803195113,
     "Intelligence only": 4.788177641426015
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 215,
   "ranked": true,
   "ranks": {
    "A": 215,
    "B": 215,
    "C": 215,
    "capability": 199
   },
   "repo": "https://huggingface.co/flymy-ai/decision-2b-preview",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 10.289709237720572,
    "B": 8.367001767819637,
    "C": 7.14563141508373
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.30001436800230297,
    "p50_s_raw": 0.07500718400115147,
    "p95_s_adjusted": 0.3043619356816635,
    "p95_s_raw": 0.07718096784083173
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "openbmb/MiniCPM5-2B with a trained LoRA adapter and pointer head (26.2M trainable parameters)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.98496799808474,
      88.16729740402266
     ],
     "I": [
      18.649938213615208,
      26.096089404676093
     ],
     "capability": [
      50.712178970237844,
      56.11601608037113
     ],
     "composite": {
      "A": [
       6.070785321244929,
       14.301729550123063
      ],
      "B": [
       4.787007736285314,
       11.888763436425752
      ],
      "C": [
       4.215823139753423,
       9.931756632029904
      ]
     },
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.77033341356835,
     "capability": 54.82290333495299,
     "intelligence": 22.875473256337628
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5853333333333334,
      "calibration": 87.98374178599691,
      "cc": 46.9700557960649,
      "cc_by_type": {
       "choice": 46.9700557960649
      },
      "cc_tiered": 47.804657407020194,
      "cc_tiered_open": 50.388518138122464,
      "cc_tiered_sealed": 45.220796675917924,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 46.9700557960649
     },
     "noul": {
      "accuracy": 0.3626666666666667,
      "calibration": 86.1418789922961,
      "cc": -27.46666666666666,
      "cc_by_type": {
       "noul": -27.46666666666666
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4266666666666667,
      "calibration": 86.18537946241204,
      "cc": 18.767847334171428,
      "cc_by_type": {
       "score": 18.767847334171428
      },
      "cc_tiered": 20.82176236199269,
      "cc_tiered_open": 23.050873664635812,
      "cc_tiered_sealed": 18.59265105934957,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 18.767847334171428
     }
    },
    "raw_on_A": {
     "calibration": 80.16146678308505,
     "capability": 51.30498083159781,
     "intelligence": 22.448494880110573
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 8.990680106093736,
    "Emphasis on Accuracy 60:20:20": 6.654472838884132,
    "Emphasis on Cost 20:20:60": 10.468173298921004,
    "Emphasis on Speed 20:60:20": 11.379624987030224,
    "Intelligence only": 4.788177641426015
   }
  },
  {
   "adapter_id": "images/decision-4b-v11/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 85.23934158755043,
    "cost": 77.61552273233744,
    "intelligence": 29.134168280172865,
    "speed": 93.91870342645993
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10557670546665254,
      "ece_score": 78.88465890666949,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 78.88465890666949,
      "score_before_time_limit": 78.88465890666949,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.4339622641509434,
      "brier": 0.11841873108613687,
      "ece": 0.04782018815907789,
      "n": 371,
      "score": 90.43596236818442,
      "score_before_time_limit": 90.43596236818442,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9701897018970189,
      "interval90_width": 0.5756549232158987,
      "n": 369,
      "nrps": 0.11671345785768895,
      "score": 86.39740348779742,
      "score_before_time_limit": 86.39740348779742,
      "time_limit_failed_share": 0,
      "top_ece": 0.07766923619318136
     }
    },
    "score": 85.23934158755043
   },
   "capability": 57.18675493386165,
   "class": "unclassified",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 921,181 input + 0 output tokens over 1,624 decisions = 567.229679803 tokens/decision. GPU-time cost USD 0.00557377254813054/1,000; old list-price cost USD 0.0170168903940887/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00557377254813054/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005573772548130538
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision 4B v1.1 (FlyMyJev, Qwen3.5-4B + LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 30.696800084915584,
    "I_sealed": 27.571536475430143,
    "base": 29.134168280172865,
    "excess": 0.5518759670471964,
    "gap": 3.1252636094854402,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 60.19131316010312,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 56.59012268008808,
       "judge": 64.40342781806196,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 1.1380262249827418,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 13.043478260869556,
       "judge": -30.000000000000004,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 30.761060869660888,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.918373499249086,
       "hard": 31.938975913174172,
       "judge": 30.29541690533948,
       "standard": 17.025040414322312
      }
     },
     "sealed|choice": {
      "cc": 57.815173715536396,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 68.07331313280615,
       "hard": 54.185520361990946,
       "judge": 56.238361266294234,
       "standard": 62.31062938785566
      }
     },
     "sealed|noul": {
      "cc": 2.4545445385650666,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 28.000000000000004,
       "hard": -16.504854368932044,
       "judge": 10.526315789473696,
       "standard": 15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 22.444891172188967,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 55.42678299206407,
       "hard": 20.963579856299262,
       "judge": 15.25087787708388,
       "standard": 19.707587836688447
      }
     }
    }
   },
   "jevbench_score": 19.516239094521907,
   "key": "decision-4b-v11",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9428571428571428,
    "decisive_rate": 0.5660377358490566,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.23934158755043,
     "cost": 63.07359395281681,
     "intelligence": 29.134168280172865,
     "speed": 93.95641941218838
    },
    "board_release": "v1.6.43",
    "capability": 57.18675493386165,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 68,
    "capability_rank": 94,
    "composite_ci95": {
     "A": [
      12.444491289404162,
      27.25430281789905
     ],
     "B": [
      10.295694887525865,
      23.807726232797425
     ],
     "C": [
      8.642007839864,
      18.926599179096563
     ]
    },
    "jevbench_score": 18.718356274892123,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 104,
    "rank": 139,
    "ranks": {
     "A": 139,
     "B": 146,
     "C": 145,
     "capability": 148
    },
    "scores": {
     "A": 18.718356274892123,
     "B": 15.883627667705545,
     "C": 12.998858524230638
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.188053295109421,
     "p50_s_raw": 0.019026647554710507,
     "p95_s_adjusted": 0.21383417290635406,
     "p95_s_raw": 0.031917086453177035
    },
    "views": {
     "Balanced 33:33:33": 16.746653513023,
     "Emphasis on Accuracy 60:20:20": 13.11195414676558,
     "Emphasis on Cost 20:20:60": 18.346342760998827,
     "Emphasis on Speed 20:60:20": 20.675164334326784,
     "Intelligence only": 9.891630033656257
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 188,
   "ranked": true,
   "ranks": {
    "A": 188,
    "B": 190,
    "C": 189,
    "capability": 180
   },
   "repo": "https://benchmarkheaven.com/jev-models/decision-4b-v11",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 19.516239094521907,
    "B": 16.337034634983198,
    "C": 13.55294381564021
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18865264891646802,
    "p50_s_raw": 0.019326324458234012,
    "p95_s_adjusted": 0.2150140054989606,
    "p95_s_raw": 0.032507002749480304
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.26750110051037,
      86.2633177195094
     ],
     "I": [
      24.784375611108008,
      34.07951912545236
     ],
     "capability": [
      52.99615856993229,
      59.129104174768116
     ],
     "composite": null,
     "seed": 30
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.23934158755043,
     "capability": 57.18675493386165,
     "intelligence": 29.134168280172865
    },
    "per_type": {
     "choice": {
      "accuracy": 0.68,
      "calibration": 78.88465890666949,
      "cc": 59.07657038924623,
      "cc_by_type": {
       "choice": 59.07657038924623
      },
      "cc_tiered": 59.00324343781976,
      "cc_tiered_open": 60.19131316010312,
      "cc_tiered_sealed": 57.815173715536396,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.07657038924623
     },
     "noul": {
      "accuracy": 0.528,
      "calibration": 90.43596236818442,
      "cc": 5.600000000000005,
      "cc_by_type": {
       "noul": 5.600000000000005
      },
      "cc_tiered": 1.7962853817739042,
      "cc_tiered_open": 1.1380262249827418,
      "cc_tiered_sealed": 2.4545445385650666,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.600000000000005
     },
     "score": {
      "accuracy": 0.47733333333333333,
      "calibration": 86.39740348779742,
      "cc": 26.584587548466242,
      "cc_by_type": {
       "score": 26.584587548466242
      },
      "cc_tiered": 26.602976020924928,
      "cc_tiered_open": 30.761060869660888,
      "cc_tiered_sealed": 22.444891172188967,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 26.584587548466242
     }
    },
    "raw_on_A": {
     "calibration": 85.6538943241602,
     "capability": 60.62425637183274,
     "intelligence": 35.59461841950529
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 17.605239889809713,
    "Emphasis on Accuracy 60:20:20": 13.419397929321498,
    "Emphasis on Cost 20:20:60": 20.300501211665924,
    "Emphasis on Speed 20:60:20": 21.447742491770843,
    "Intelligence only": 9.891630033656257
   }
  },
  {
   "adapter_id": "images/decision-4b-v12/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 88.42488519968094,
    "cost": 77.61552273233744,
    "intelligence": 32.40310972936693,
    "speed": 93.8753186824915
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0672265434886423,
      "ece_score": 86.55469130227154,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 86.55469130227154,
      "score_before_time_limit": 86.55469130227154,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.41239892183288407,
      "brier": 0.12038668872149434,
      "ece": 0.03739565815349834,
      "n": 371,
      "score": 92.52086836930033,
      "score_before_time_limit": 92.52086836930033,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.975609756097561,
      "interval90_width": 0.5718157181571816,
      "n": 369,
      "nrps": 0.10515132123479526,
      "score": 86.19909592747098,
      "score_before_time_limit": 86.19909592747098,
      "time_limit_failed_share": 0,
      "top_ece": 0.08543338010789257
     }
    },
    "score": 88.42488519968094
   },
   "capability": 60.41399746452394,
   "class": "unclassified",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 921,181 input + 0 output tokens over 1,624 decisions = 567.229679803 tokens/decision. GPU-time cost USD 0.00557377254813054/1,000; old list-price cost USD 0.0170168903940887/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00557377254813054/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005573772548130538
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision 4B v1.2 (FlyMyJev, Qwen3.5-4B + LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 32.99062805982875,
    "I_sealed": 31.81559139890512,
    "base": 32.40310972936693,
    "excess": -1.398350981514613,
    "gap": 1.175036660923631,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 61.39828743553336,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 50.92796476879523,
       "judge": 68.35860250494396,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 6.031055900621121,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": 30.434782608695656,
       "judge": -30.000000000000004,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 31.542540843331764,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.77190646723929,
       "hard": 38.72644314542413,
       "judge": 25.494038632486827,
       "standard": 17.13280674346067
      }
     },
     "sealed|choice": {
      "cc": 58.83327326304771,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 68.07331313280615,
       "hard": 56.730769230769226,
       "judge": 56.238361266294234,
       "standard": 62.31062938785566
      }
     },
     "sealed|noul": {
      "cc": 8.040955184350867,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 28.000000000000004,
       "hard": -8.737864077669899,
       "judge": 13.157894736842103,
       "standard": 23.943661971830977
      }
     },
     "sealed|score": {
      "cc": 28.5725457493168,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 58.99125190199095,
       "hard": 23.870346180703184,
       "judge": 26.811123175914563,
       "standard": 25.409725670310312
      }
     }
    }
   },
   "jevbench_score": 25.567225216905292,
   "key": "decision-4b-v12",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.944954128440367,
    "decisive_rate": 0.5876010781671159,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.42488519968094,
     "cost": 63.07359395281681,
     "intelligence": 32.40310972936693,
     "speed": 93.92936299294371
    },
    "board_release": "v1.6.43",
    "capability": 60.41399746452394,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 59,
    "capability_rank": 84,
    "composite_ci95": {
     "A": [
      15.392618056749317,
      34.34955627190713
     ],
     "B": [
      12.888673039303178,
      30.38967783428378
     ],
     "C": [
      10.689318094964802,
      23.853858522157736
     ]
    },
    "jevbench_score": 24.463556699266523,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 82,
    "rank": 115,
    "ranks": {
     "A": 115,
     "B": 125,
     "C": 130,
     "capability": 130
    },
    "scores": {
     "A": 24.463556699266523,
     "B": 21.09790572169507,
     "C": 16.988581041157314
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.18829862689599394,
     "p50_s_raw": 0.019149313447996974,
     "p95_s_adjusted": 0.2148901690961793,
     "p95_s_raw": 0.032445084548089656
    },
    "views": {
     "Balanced 33:33:33": 21.964945240595316,
     "Emphasis on Accuracy 60:20:20": 17.633890573157025,
     "Emphasis on Cost 20:20:60": 23.575828038111915,
     "Emphasis on Speed 20:60:20": 26.69802362301605,
     "Intelligence only": 13.608807339398107
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 177,
   "ranked": true,
   "ranks": {
    "A": 177,
    "B": 180,
    "C": 179,
    "capability": 163
   },
   "repo": "https://benchmarkheaven.com/jev-models/decision-4b-v12",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 25.567225216905292,
    "B": 21.745547935399053,
    "C": 17.755017511739794
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18921992308460175,
    "p50_s_raw": 0.01960996154230088,
    "p95_s_adjusted": 0.2165216211695224,
    "p95_s_raw": 0.0332608105847612
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.81349929936302,
      88.39912781374252
     ],
     "I": [
      26.905961082898493,
      37.32153047837871
     ],
     "capability": [
      55.53187740606071,
      61.760259132085466
     ],
     "composite": null,
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.42488519968094,
     "capability": 60.41399746452394,
     "intelligence": 32.40310972936693
    },
    "per_type": {
     "choice": {
      "accuracy": 0.688,
      "calibration": 86.55469130227154,
      "cc": 60.099656129515054,
      "cc_by_type": {
       "choice": 60.099656129515054
      },
      "cc_tiered": 60.11578034929053,
      "cc_tiered_open": 61.39828743553336,
      "cc_tiered_sealed": 58.83327326304771,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.099656129515054
     },
     "noul": {
      "accuracy": 0.5493333333333333,
      "calibration": 92.52086836930033,
      "cc": 9.866666666666667,
      "cc_by_type": {
       "noul": 9.866666666666667
      },
      "cc_tiered": 7.036005542485994,
      "cc_tiered_open": 6.031055900621121,
      "cc_tiered_sealed": 8.040955184350867,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 9.866666666666667
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 86.19909592747098,
      "cc": 30.992722913904714,
      "cc_by_type": {
       "score": 30.992722913904714
      },
      "cc_tiered": 30.057543296324283,
      "cc_tiered_open": 31.542540843331764,
      "cc_tiered_sealed": 28.5725457493168,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.992722913904714
     }
    },
    "raw_on_A": {
     "calibration": 81.8292718059759,
     "capability": 58.67224928043624,
     "intelligence": 35.515226754896574
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 23.1619022284358,
    "Emphasis on Accuracy 60:20:20": 18.08405312239437,
    "Emphasis on Cost 20:20:60": 26.194773973113776,
    "Emphasis on Speed 20:60:20": 27.739136631866614,
    "Intelligence only": 13.608807339398107
   }
  },
  {
   "adapter_id": "images/decision-fast/spec.json:flymy_v15.FlyMyPackage",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "FlyMy.AI (@denti)",
   "axes": {
    "calibration": 75.07737238688318,
    "cost": 86.6432611441359,
    "intelligence": 6.110691078442272,
    "speed": 91.47223870429099
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1702515239808779,
      "ece_score": 65.94969520382442,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 65.94969520382442,
      "score_before_time_limit": 65.94969520382442,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.8625336927223719,
      "brier": 0.2408499885802099,
      "ece": 0.11364094419260821,
      "n": 371,
      "score": 77.27181116147835,
      "score_before_time_limit": 77.27181116147835,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.959349593495935,
      "interval90_width": 0.8803071364046972,
      "n": 369,
      "nrps": 0.17723101114191298,
      "score": 82.01061079534679,
      "score_before_time_limit": 82.01061079534679,
      "time_limit_failed_share": 0,
      "top_ece": 0.09127838647557562
     }
    },
    "score": 75.07737238688318
   },
   "capability": 40.59403173266273,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.6B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 748,805 input + 0 output tokens over 1,624 decisions = 461.08682266 tokens/decision. GPU-time cost USD 0.00278756259651849/1,000; old list-price cost USD 0.00461086822660099/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00278756259651849/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0027875625965184925
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision Fast (FlyMy.AI, v53a)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 5.162912545807329,
    "I_sealed": 7.058469611077215,
    "base": 6.110691078442272,
    "excess": -4.468944707708131,
    "gap": -1.8955570652698865,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 15.371451872027388,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.945945945945947,
       "hard": 3.743315508021402,
       "judge": 24.851680949241928,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.28571428571429,
       "hard": -100,
       "judge": -90,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0.1172857653946,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 37.515372511743564,
       "hard": -11.188275382721603,
       "judge": -4.115171438893084,
       "standard": 10.37805049488405
      }
     },
     "sealed|choice": {
      "cc": 21.175408833231646,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 32.15579040721308,
       "hard": 4.298642533936631,
       "judge": 27.374301675977662,
       "standard": 40.14041138071193
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -16.000000000000004,
       "hard": -88.3495145631068,
       "judge": -86.8421052631579,
       "standard": -83.09859154929578
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 22.391298609400668,
       "hard": -12.696561777971827,
       "judge": 3.397122091897453,
       "standard": -2.490577451116116
      }
     }
    }
   },
   "jevbench_score": 0.29956154664320095,
   "key": "decision-fast",
   "last_measured_on": "2026-10-05",
   "licence": "Apache-2.0 notices on the included code and the pinned base; the weights are an evaluation preview under EVALUATION-PERMISSION.md, not a cleared commercial release",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9019607843137255,
    "decisive_rate": 0.13746630727762804,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 75.07737238688318,
     "cost": 80.08651867745036,
     "intelligence": 6.110691078442272,
     "speed": 91.472185793536
    },
    "board_release": "v1.6.43",
    "capability": 40.59403173266273,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 112,
    "capability_rank": 140,
    "composite_ci95": {
     "A": [
      0.11705708633042304,
      0.8235356910346963
     ],
     "B": [
      0.07857381222727884,
      0.5860129975515587
     ],
     "C": [
      0.08128964328501598,
      0.5718997854407614
     ]
    },
    "jevbench_score": 0.2981489585308357,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 166,
    "rank": 210,
    "ranks": {
     "A": 210,
     "B": 210,
     "C": 210,
     "capability": 204
    },
    "scores": {
     "A": 0.2981489585308357,
     "B": 0.20514889729513758,
     "C": 0.2070478878686359
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2653681320138276,
     "p50_s_raw": 0.05768406600691378,
     "p95_s_adjusted": 0.26849280574824663,
     "p95_s_raw": 0.059246402874123305
    },
    "views": {
     "Balanced 33:33:33": 0.23953342944478384,
     "Emphasis on Accuracy 60:20:20": 0.14519180403822335,
     "Emphasis on Cost 20:20:60": 0.3522038654235207,
     "Emphasis on Speed 20:60:20": 0.35744386580296805,
     "Intelligence only": 0.09127061519323529
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 251,
   "ranked": true,
   "ranks": {
    "A": 251,
    "B": 251,
    "C": 251,
    "capability": 246
   },
   "repo": "https://huggingface.co/flymy-ai/decision-fast-preview",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.29956154664320095,
    "B": 0.20568278917200367,
    "C": 0.20802885183555622
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2652455839328468,
    "p50_s_raw": 0.05762279196642339,
    "p95_s_adjusted": 0.2686135815223679,
    "p95_s_raw": 0.05930679076118395
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3-0.6B-Base with a trained LoRA adapter and pointer head (10.6M trainable parameters)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      69.51262824417705,
      78.1703871756354
     ],
     "I": [
      4.394274632210615,
      8.794694293942209
     ],
     "capability": [
      37.53859057048145,
      42.929975748521436
     ],
     "composite": null,
     "seed": 32
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 75.07737238688318,
     "capability": 40.59403173266273,
     "intelligence": 6.110691078442272
    },
    "per_type": {
     "choice": {
      "accuracy": 0.37333333333333335,
      "calibration": 65.94969520382442,
      "cc": 19.85828367894051,
      "cc_by_type": {
       "choice": 19.85828367894051
      },
      "cc_tiered": 18.273430352629518,
      "cc_tiered_open": 15.371451872027388,
      "cc_tiered_sealed": 21.175408833231646,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 19.85828367894051
     },
     "noul": {
      "accuracy": 0.12266666666666666,
      "calibration": 77.27181116147835,
      "cc": -75.46666666666667,
      "cc_by_type": {
       "noul": -75.46666666666667
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.31733333333333336,
      "calibration": 82.01061079534679,
      "cc": -1.6484760263063114,
      "cc_by_type": {
       "score": -1.6484760263063114
      },
      "cc_tiered": 0.0586428826973,
      "cc_tiered_open": 0.1172857653946,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 74.21190941158237,
     "capability": 40.84165753605771,
     "intelligence": 7.471405660533052
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.24074952160330113,
    "Emphasis on Accuracy 60:20:20": 0.14545902402862795,
    "Emphasis on Cost 20:20:60": 0.35697628895956823,
    "Emphasis on Speed 20:60:20": 0.3590678295538737,
    "Intelligence only": 0.09127061519323529
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "milliseconds.ai (Baptiste Laget)",
   "axes": {
    "calibration": 79.15,
    "cost": 58.33,
    "intelligence": 13.11,
    "speed": 92.87
   },
   "calibration": {
    "score": 79.15
   },
   "capability": 46.13,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     0.89,
     8.58
    ],
    "B": [
     0.64,
     7.01
    ],
    "C": [
     0.62,
     5.96
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.04/M input, 0/M output (spec decision-machine-1.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.02449299492385787
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "decision-machine-1 (milliseconds.ai)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 10.71,
    "I_sealed": 10.36,
    "base": 13.11,
    "per_type_split": {
     "open|choice": {
      "cc": 32.1365,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 33.8378,
       "hard": 13.1802,
       "judge": 36.7172,
       "standard": 62.3273
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.2857,
       "hard": -47.8261,
       "judge": -50,
       "standard": -66.6667
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 37.3934,
       "hard": -8.8505,
       "judge": 13.5155,
       "standard": -60.8938
      }
     },
     "sealed|choice": {
      "cc": 31.0657,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 38.8323,
       "hard": 34.073,
       "judge": 27.6382,
       "standard": 26.309
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 5.8824,
       "hard": -72.7273,
       "judge": -90,
       "standard": 0
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 8.9152,
       "hard": -27.1485,
       "judge": -46.9134,
       "standard": 35.2328
      }
     }
    }
   },
   "jevbench_score": 2.35,
   "key": "decision-machine-1",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary API, closed weights",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 42,
    "axes": {
     "calibration": 79.15,
     "cost": 58.33,
     "intelligence": 13.11,
     "speed": 92.97
    },
    "board_release": "v1.6.43",
    "capability": 46.13,
    "capability_api_board_rank": 27,
    "capability_open_board_rank": null,
    "capability_rank": 126,
    "composite_ci95": {
     "A": [
      0.89,
      8.58
     ],
     "B": [
      0.64,
      7.01
     ],
     "C": [
      0.62,
      5.96
     ]
    },
    "jevbench_score": 2.35,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 196,
    "ranks": {
     "A": 196,
     "B": 196,
     "C": 196,
     "capability": 185
    },
    "scores": {
     "A": 2.35,
     "B": 1.78,
     "C": 1.63
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.167,
     "p50_s_raw": 0.167,
     "p95_s_adjusted": 0.303,
     "p95_s_raw": 0.303
    },
    "views": null
   },
   "rank": 236,
   "ranked": true,
   "ranks": {
    "A": 236,
    "B": 236,
    "C": 236,
    "capability": 224
   },
   "repo": "https://www.milliseconds.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 2.35,
    "B": 1.78,
    "C": 1.63
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.171,
    "p50_s_raw": 0.171,
    "p95_s_adjusted": 0.302,
    "p95_s_raw": 0.302
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "9fb20f7a1415fa931bb212eade8b88af31affff7800a8f45949b8b519e3bf2be"
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "vLLM Semantic Router Team (vllm-sr)",
   "axes": {
    "calibration": 87.47631981746561,
    "cost": 68.1128992661464,
    "intelligence": 35.82270435669186,
    "speed": 89.53277608263193
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05090298449957041,
      "ece_score": 89.81940310008592,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 89.81940310008592,
      "score_before_time_limit": 89.81940310008592,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.46648793565683644,
      "brier": 0.14462735182051856,
      "ece": 0.08496977140192581,
      "n": 373,
      "score": 83.00604571961485,
      "score_before_time_limit": 83.00604571961485,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9731182795698925,
      "interval90_width": 0.5766129032258064,
      "n": 372,
      "nrps": 0.09926110585728637,
      "score": 89.60351063269607,
      "score_before_time_limit": 89.60351063269607,
      "time_limit_failed_share": 0,
      "top_ece": 0.054334340744396026
     }
    },
    "score": 87.47631981746561
   },
   "capability": 61.64951208707873,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 776,349 input + 0 output tokens over 1,485 decisions = 522.793939394 tokens/decision. GPU-time cost USD 0.0115585500474062/1,000; old list-price cost USD 0.0522793939393939/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0115585500474062/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0517566
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011558550047406203
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision 2.0 Lux 9B (vLLM-SR, Qwen3.5-9B lineage)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Authors' own in-process runtime behind the evaluator's loopback shim (verbatim from the earlier v1.5 measurement), serial requests.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.41506937866855,
    "I_sealed": 30.230339334715172,
    "base": 35.82270435669186,
    "excess": 1.3452032735976651,
    "gap": 11.184730043953376,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 65.97342471411105,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 64.1396665618119,
       "judge": 72.31377719182598,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 10.641821946169772,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -4.347826086956519,
       "judge": 0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 47.62996147572483,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 57.01722460619536,
       "hard": 42.15889431879225,
       "judge": 48.06315937576442,
       "standard": 53.22866737429535
      }
     },
     "sealed|choice": {
      "cc": 59.49041617562122,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 53.16294668671215,
       "judge": 55.92556317335946,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": -9.090909090909093,
       "judge": -41.53846153846153,
       "standard": 16.417910447761198
      }
     },
     "sealed|score": {
      "cc": 31.200601828524306,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 52.52957611300084,
       "hard": 27.31494795658824,
       "judge": 23.847538197285967,
       "standard": 39.33701787701567
      }
     }
    }
   },
   "jevbench_score": 31.492390122627054,
   "key": "decision2-lux-9b",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "vllm-sr/Decision-2.0-Lux-9B@78bf3c03d9147aeb30b641edfe0e30ed04887ca5",
   "noul_decisive": {
    "acc_among_decisive": 0.9447236180904522,
    "decisive": 199,
    "decisive_rate": 0.5335120643431636,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.47631981746561,
     "cost": 48.45008367155653,
     "intelligence": 35.82270435669186,
     "speed": 89.93254869015362
    },
    "board_release": "v1.6.43",
    "capability": 61.64951208707873,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 56,
    "capability_rank": 81,
    "composite_ci95": {
     "A": [
      20.401360460654523,
      34.47961583493885
     ],
     "B": [
      18.006770417598776,
      31.551034699005438
     ],
     "C": [
      14.167611431010082,
      23.944177663151983
     ]
    },
    "jevbench_score": 27.113058507774674,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": 75,
    "rank": 108,
    "ranks": {
     "A": 108,
     "B": 108,
     "C": 120,
     "capability": 125
    },
    "scores": {
     "A": 27.113058507774674,
     "B": 24.336976546681964,
     "C": 18.828512852621298
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2866953719989397,
     "p50_s_raw": 0.06834768599946983,
     "p95_s_adjusted": 0.3542619008774636,
     "p95_s_raw": 0.10213095043873181
    },
    "views": {
     "Balanced 33:33:33": 24.230269361637582,
     "Emphasis on Accuracy 60:20:20": 20.8638737710436,
     "Emphasis on Cost 20:20:60": 23.871066823962767,
     "Emphasis on Speed 20:60:20": 29.419879270169634,
     "Intelligence only": 17.265698468557986
    }
   },
   "rank": 160,
   "ranked": true,
   "ranks": {
    "A": 160,
    "B": 162,
    "C": 162,
    "capability": 156
   },
   "repo": "https://huggingface.co/vllm-sr/Decision-2.0-Lux-9B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 31.492390122627054,
    "B": 27.56369520937155,
    "C": 21.86971536293545
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2906017030240037,
    "p50_s_raw": 0.07030085151200183,
    "p95_s_adjusted": 0.3831988849298795,
    "p95_s_raw": 0.11659944246493978
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 65.38134362243356,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 29.477364200633982
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 65.38134362243356
      },
      "calibration_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 4.099569847308422,
       "cc_by_type": {
        "choice": 30.010604453870634,
        "noul": -28.000000000000004,
        "score": 10.288105088054634
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 4.099569847308422
      },
      "composite_gen": {
       "accuracy": 0.6237623762376238,
       "cc": 26.33004954263009,
       "cc_by_type": {
        "choice": 77.23577235772358,
        "noul": -48.14814814814815,
        "score": 49.90252441831484
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 26.33004954263009
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 62.654116824285275,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 67.96235047285583
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 62.654116824285275
      },
      "jaggedness": {
       "accuracy": 0.4,
       "cc": -3.3916129517076663,
       "cc_by_type": {
        "choice": 30.702598652550538,
        "noul": -56.75675675675676,
        "score": 15.879319249083224
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.8146964856230032,
       "cc": 67.14156045601494,
       "cc_by_type": {
        "choice": 77.01857885004843,
        "noul": 67.01030927835052,
        "score": 57.395793239645876
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 67.14156045601494
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -11.681952585069832,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -100,
        "score": 14.43867832726472
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6363636363636364,
       "cc": 33.066005227920634,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": -11.111111111111116,
        "score": 38.67560869901385
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 33.066005227920634
      },
      "negation_pair_gen": {
       "accuracy": 0.4883720930232558,
       "cc": 2.67222418279357,
       "cc_by_type": {
        "choice": 56.66666666666668,
        "noul": -60,
        "score": 11.350005881714031
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 2.67222418279357
      },
      "option_order": {
       "accuracy": 0.8958333333333334,
       "cc": 86.68442077230361,
       "cc_by_type": {
        "choice": 86.68442077230361
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 86.68442077230361
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 74.77426699693255,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 80,
        "score": 71.3498280178247
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 74.77426699693255
      },
      "usecase": {
       "accuracy": 0.509090909090909,
       "cc": 26.66485457151172,
       "cc_by_type": {
        "choice": 47.30392156862747,
        "noul": -4.1666666666666625,
        "score": 36.85730881257436
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 26.66485457151172
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 9.056945780385973,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0,
        "score": -15.686305515984934
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 9.056945780385973
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 17.326598840370945,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": 26.787784232326363
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 17.326598840370945
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 67.11320598246611,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 50.233423257132834
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 67.11320598246611
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 33.09357426464621,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 99.46694253323099
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 33.09357426464621
      },
      "en": {
       "accuracy": 0.6046915725456126,
       "cc": 31.4531064905039,
       "cc_by_type": {
        "choice": 58.701992897328545,
        "noul": 0,
        "score": 35.657326574183145
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 31.4531064905039
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 63.51130608648939,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 10.10977470318759
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 63.51130608648939
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 19.74093935418468,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 25.34303664178902
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 19.74093935418468
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 40.99009270088603,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 65.32321927912866
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 40.99009270088603
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 18.916283367574735,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -50,
        "score": 23.508626639037043
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 18.916283367574735
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 74.61768112255443,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 23.853043367663275
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 74.61768112255443
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 14.22770772278479,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": -19.999999999999996,
        "score": 34.16842896660695
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 14.22770772278479
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -9.761328844855484,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 15.160457909878
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 72.09322796696891,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 82.9463505675734
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 72.09322796696891
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 9.435501837148886,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 28.306505511446655
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.435501837148886
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 39.71775347984494,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": -19.999999999999996,
        "score": 46.77069590292883
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 39.71775347984494
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 66.8390486833019,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 42.94495983894516
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 66.8390486833019
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 3.6562373029894957,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 47.564456589819535
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 3.6562373029894957
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 24.52332791887444,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50,
        "score": -54.69088580859407
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 24.52332791887444
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 69.66005480942651,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 41.23822894440856
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 69.66005480942651
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 6.234913945334711,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -47.961924830662525
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 6.234913945334711
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7026666666666667,
       "cc": 61.595131938996154,
       "cc_by_type": {
        "choice": 61.595131938996154
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 61.595131938996154
      },
      "noul": {
       "accuracy": 0.5013333333333333,
       "cc": 0.2666666666666595,
       "cc_by_type": {
        "noul": 0.2666666666666595
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.2666666666666595
      },
      "score": {
       "accuracy": 0.5573333333333333,
       "cc": 35.76324809476839,
       "cc_by_type": {
        "score": 35.76324809476839
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 35.76324809476839
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -21.716999577358354,
       "cc_by_type": {
        "choice": 52,
        "noul": -100,
        "score": -17.150998732075063
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 30.171019428261456,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 11.111111111111116,
        "score": 16.669027919014855
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 30.171019428261456
      },
      "customer_support": {
       "accuracy": 0.9285714285714286,
       "cc": 66.97464414728488,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 50.923932441854646
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 66.97464414728488
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -15.449793509057187,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": -19.44879573184992
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 23.250864501749692,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 32.071434084959215
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 23.250864501749692
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -65.29273959291613,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -66.66666666666667,
        "score": -141.16123764667293
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 21.627541374440955,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -42.85714285714286,
        "score": 37.15153168634807
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 21.627541374440955
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": 10.26483205645294,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50,
        "score": 53.96522787667589
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 10.26483205645294
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 9.522250670781048,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 51.066752012343144
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 9.522250670781048
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 21.435076662665292,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 0,
        "score": 28.545058681786028
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 21.435076662665292
      },
      "lead_generation": {
       "accuracy": 0.56,
       "cc": 33.301792034961935,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 22.644394192741114
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 33.301792034961935
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -31.99789804370859,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": -33.62810273327632
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5185185185185185,
       "cc": 25.452132963671094,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 30.90534322882519
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 25.452132963671094
      },
      "model_routing": {
       "accuracy": 0.8341013824884793,
       "cc": 67.04627110748747,
       "cc_by_type": {
        "choice": 84.80519480519482,
        "noul": 72.72727272727273,
        "score": 43.60634578999484
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 67.04627110748747
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 6.899543267261227,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 66.76388891886621
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 6.899543267261227
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 50.511962779197546,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 51.62873884827046
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.511962779197546
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 1.2750601937910584,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50,
        "score": -16.618169172321416
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 1.2750601937910584
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 12.33326926457949,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 42.79690924301383
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 12.33326926457949
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 91.439262104532,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 84.99394289722593
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 91.439262104532
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.99931384938026,
      87.44256636228893
     ],
     "I": [
      31.9535738870049,
      39.72745829130512
     ],
     "capability": [
      57.416494415371886,
      62.68561954122301
     ],
     "composite": null,
     "seed": 28
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.47631981746561,
     "capability": 61.64951208707873,
     "intelligence": 35.82270435669186
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7026666666666667,
      "calibration": 89.81940310008592,
      "cc": 61.595131938996154,
      "cc_by_type": {
       "choice": 61.595131938996154
      },
      "cc_tiered": 62.731920444866134,
      "cc_tiered_open": 65.97342471411105,
      "cc_tiered_sealed": 59.49041617562122,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.595131938996154
     },
     "noul": {
      "accuracy": 0.5013333333333333,
      "calibration": 83.00604571961485,
      "cc": 0.2666666666666595,
      "cc_by_type": {
       "noul": 0.2666666666666595
      },
      "cc_tiered": 5.320910973084886,
      "cc_tiered_open": 10.641821946169772,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.2666666666666595
     },
     "score": {
      "accuracy": 0.5573333333333333,
      "calibration": 89.60351063269607,
      "cc": 35.76324809476839,
      "cc_by_type": {
       "score": 35.76324809476839
      },
      "cc_tiered": 39.41528165212457,
      "cc_tiered_open": 47.62996147572483,
      "cc_tiered_sealed": 31.200601828524306,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.76324809476839
     }
    },
    "raw_on_A": {
     "calibration": 81.01141299701578,
     "capability": 59.06449579734621,
     "intelligence": 37.11757859767665
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 28.64121272537459,
    "Emphasis on Accuracy 60:20:20": 23.418040630880434,
    "Emphasis on Cost 20:20:60": 30.87413842106689,
    "Emphasis on Speed 20:60:20": 33.723992654280885,
    "Intelligence only": 18.38802552409063
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bynn Intelligence (Hugging Face)",
   "axes": {
    "calibration": 77.47576560172445,
    "cost": 86.77310580145102,
    "intelligence": 26.810082223231646,
    "speed": 92.88928290241705
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11639344262295086,
      "ece_score": 76.72131147540982,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 76.72131147540982,
      "score_before_time_limit": 76.72131147540982,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.37801608579088475,
      "brier": 0.18461608579088468,
      "ece": 0.12230563002680964,
      "n": 373,
      "score": 75.53887399463808,
      "score_before_time_limit": 75.53887399463808,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8844086021505376,
      "interval90_width": 0.4422043010752688,
      "n": 372,
      "nrps": 0.1265502464157705,
      "score": 80.16711133512545,
      "score_before_time_limit": 80.16711133512545,
      "time_limit_failed_share": 0,
      "top_ece": 0.1350537634408602
     }
    },
    "score": 77.47576560172445
   },
   "capability": 52.14292391247805,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 677,924 input + 0 output tokens over 1,485 decisions = 456.514478114 tokens/decision. GPU-time cost USD 0.00275991987066432/1,000; old list-price cost USD 0.00913028956228956/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00275991987066432/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.009038986666666667
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0027599198706643154
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "DecisionNode Edge 0.9.2",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.574/h as recorded by the runner (300 GB disk configured). One VM for both rows back to back with verified node shredding after each row and a final shred. The author's server (127.0.0.1:8000) in a --network none --read-only container per row; driver requests serial over loopback. Read-only code, model and input, empty credential variables, HF offline flags; cut proof inside the container before any sealed input (only loopback, no default route, DNS and TCP fail). Measured 24 Oct 2026 (reservation 05:33:57Z, VM created 05:35:01Z, teardown confirmed 05:43:05Z); this row's measurement container 91 s (run cap 112 s), scored loop 49.329 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 395ad44a0408cc796bca562f7ef9b47eb20fab6320c50273969a0ce0efd9134f. Node-shred receipt verified for each row; exact-id VM and volume teardown confirms provider absent and guard closed. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 33.72323159403544,
    "I_sealed": 19.896932852427852,
    "base": 26.810082223231646,
    "excess": 3.9867719712518763,
    "gap": 13.826298741607587,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 47.260855396922054,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 45.265806857502355,
       "judge": 56.49307844429796,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 18.697722567287787,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": 30.434782608695656,
       "judge": -9.999999999999998,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 35.211116817896475,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.45454545454546,
       "hard": 23.990740740740733,
       "judge": 32.16949152542372,
       "standard": 50.59259259259259
      }
     },
     "sealed|choice": {
      "cc": 40.910318515951026,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 61.88055908513342,
       "hard": 34.94853706487798,
       "judge": 39.17727717923606,
       "standard": 44.94832313857837
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -3.4482758620689613,
       "hard": -9.090909090909093,
       "judge": -7.692307692307687,
       "standard": 16.417910447761198
      }
     },
     "sealed|score": {
      "cc": 18.78048004133253,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 32.99122807017544,
       "hard": 11.535211267605682,
       "judge": 16.16666666666665,
       "standard": 30.08636363636361
      }
     }
    }
   },
   "jevbench_score": 15.863513503818993,
   "key": "decisionnode-edge-0.9.2",
   "last_measured_on": "2026-10-24",
   "licence": "AGPL-3.0-only for the weights and the decisionnode_edge runtime (LICENSING.md, LICENSE, manifest licence agpl-3.0; commercial licence offered separately by Bynn Intelligence Inc.); base model Qwen/Qwen3.5-2B-Base Apache-2.0 (LICENSE-base); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "bynn-intelligence-com/decisionnode-edge@0823eb7134172e1c8c8daaa7244a6cc77c5e5ac9",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.8275862068965517,
    "decisive": 232,
    "decisive_rate": 0.6219839142091153,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 204,
   "ranked": true,
   "ranks": {
    "A": 204,
    "B": 206,
    "C": 204,
    "capability": 207
   },
   "repo": "https://huggingface.co/bynn-intelligence-com/decisionnode-edge",
   "scores": {
    "A": 15.863513503818993,
    "B": 13.093032912410294,
    "C": 11.016328822096522
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21247871499999746,
    "p50_s_raw": 0.031239357499998732,
    "p95_s_adjusted": 0.24196707719999608,
    "p95_s_raw": 0.04598353859999804
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run unmodified: bynn-intelligence-com/decisionnode-edge at 0823eb7134172e1c8c8daaa7244a6cc77c5e5ac9 (tag v0.9.2) with the authors' PyPI package decisionnode-edge 0.9.2 (wheel sha256 dfb0b48c1233c32a106d8eee4d6c320dc3454026a1119851787f345b2f5c6317, sdist sha256 2f1da8de0be9edb3a58c3804dc024cede75b53dcea9aa82a42645e1a189ccbd0): 11 Python modules, installed from the hash-pinned wheel with --no-deps --no-index under a network-free image build; the 11 modules equal (git blob oid) runtime/src/decisionnode_edge in both Hub repos at their pins. Only the 10 data files of the pinned revision were mounted read-only at /model (config.json, decisionnode_edge.json, calibration.json, box_calibration.json, tokenizer.json, tokenizer_config.json, preprocessor_config.json, video_preprocessor_config.json, model.safetensors, readout.safetensors); no Python, GGUF or shell file of the Hub repo was fetched. Command: env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 USE_HUB_KERNELS=NO DECISIONNODE_EDGE_BACKEND=torch PYTHONDONTWRITEBYTECODE=1 python3 -m decisionnode_edge serve /model --host 127.0.0.1 --port 8000 --backend torch. PyTorch bf16 reference backend, one request at a time, no optional kernels (flash-linear-attention, causal-conv1d, Hub kernels, openvino and mlx absent; checked in the image). Image: vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 used as the CUDA 13.0 / PyTorch base (vLLM 0.30.0 is installed in it but was not used: the author's default --backend auto would select vllm there, so torch was set on the command line and in the environment, and /healthz reported backend torch after the sealed run), derived image sha256:b1f31eab61d184033ae78a92aa778c7faa2c909f8566725b4716d7803535dd47 built offline from three hash-pinned wheels (decisionnode_edge 0.9.2, transformers 5.18.0 sha256 d79e5a515a572ee3deb33eb0d578be47d91e70c75d54cf21bf6eede6edef14a9, huggingface_hub 1.33.0 sha256 04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899). Python 3.12.3, torch 2.13.0+cu130, transformers 5.18.0, tokenizers 0.23.2, safetensors 0.8.0, numpy 2.2.6. Code gate: cleared by the benchmark owner on the disclosed patterns. DISCLOSED PATTERNS: dormant subprocess start of a llama-server child in the llamacpp backend (backends.py:353), reachable only with --backend llamacpp; GGUF build subprocesses (build.py:134,136), a file imported by nothing and run only as python -m decisionnode_edge.build; Hub-capable from_pretrained loaders (model.py:49,92,235) that receive the local hash-checked model folder, under HF_HUB_OFFLINE=1, TRANSFORMERS_OFFLINE=1 and a container without network. No runtime download, trust_remote_code, string eval/exec, pickle, telemetry or request-body logging in the author code. The source and runner were reviewed before the run, with the required conditions enforced. A leftover .cache directory that would have stopped the run before any sealed item was removed and the fix rechecked; the exact device selection change was also reviewed. Weights: AGPL-3.0, public, ungated, 2,213,372,736 bf16 parameters, 10 data files of 4,449,009,719 bytes in total hash-checked file by file (verdict OK); prereg manifest sha256 948477ee75aa7f7f53f65be22babda905c06ce68fa664d3aa91618ba1385d81a. The Hub main branch of the repo is one commit ahead of the pinned tag (7706b508), with zero file differences against the pinned tag commit (added 0, removed 0, changed 0 in our file-by-file comparison); the measurement stays at the pinned commit 0823eb71. FIRST MEASUREMENT of DecisionNode Edge 0.9.2 (our own intake; filed as GitHub issue 231 by the account @bynnit on 23 Oct 2026). No DecisionNode row exists on the live JevBench board (v1.6.42, checked 24 Oct 2026). AUTHOR'S OWN SERVER on our rented hardware, PyTorch bf16 reference backend, one request at a time. NATIVE OUTPUTS: probabilities over the item's options from the model's readout head with the author's calibration applied by the server (choice argmax and probabilities; score level, expectation and probabilities; noul P(true) from the server's truth answer reported as a noul); the server rounds probabilities to 2 decimals; nothing is generated. NAMING NOTE: the package, the PyPI release and the Hub tags are 0.9.2, but the served model name reads decisionnode-edge-0.9.1 (/healthz, /v1/models and every response): the name string comes from decisionnode_edge.json in the model folder. the source review of 23 Oct 2026 found model.safetensors, readout.safetensors, config.json, tokenizer.json and calibration.json byte-identical across the tags v0.9.1 and v0.9.2; v0.9.2 adds the /v1/systemone server, box answers and documentation. The measured model tensors are therefore the 0.9.1 weights, served by the 0.9.2 runtime. PYTORCH PATH: the author's default backend selection would have used vLLM in this image; we forced the PyTorch reference path, without the optional flash-linear-attention and causal-conv1d kernels. The author's README says each backend agrees with the PyTorch reference on at least 98% of top answers and that the PyTorch path without the kernels gives the same answers about twice as slowly on long inputs; we did not test either statement. The README's timing (9.5 ms at about 500 tokens, vLLM backend, RTX PRO 6000) does not transfer to this PyTorch measurement. AUTHOR EXPOSURE (volunteered by the author in the submission and recorded in the source review; not verified by us): no JevBench item is in the training data (every training record was checked against the public items by exact text and overlapping word windows, and matches were dropped), but the public items were one of the authors' development suites: they read scores on them while comparing training runs, which is model selection against the public split. No overlap or contamination audit was run by us. LICENCE NOTE: AGPL-3.0 for the weights and the runtime; Bynn Intelligence Inc. also offers a separate commercial licence (LICENSING.md); the licence-attribution requirement has not been resolved. NOT ANSWERED, counted wrong: 15 items exceed the author's 16,384-token context limit (HTTP 400, error_type max_tokens_exceeded), exactly the 15 standard very long items (76,309-80,466 tokens by our first-party offline count) at positions in the pre-registered long-item set, as pre-registered by id and position; never truncated. Answered: 1,485 of the 1,500 items sent, HTTP 200 (status counts 200: 1,485, 400: 15); the longest answered item was 1,794 tokens. 10 s RULE: 0 decisions over 10 s, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); maximum 0.111 s over the 1,500 requests (median 0.0312 s, p95 0.0460 s). DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0 (compared on the server's 2-decimal outputs). CONTEXT: author-imposed limit 16,384 tokens; no truncation. Pre-sealed synthetic probes (not benchmark items): a 16,384-token request was accepted (0.76 s), requests of 16,385 and 82,000 tokens were refused with HTTP 400 and the server stayed healthy. The vendor's reported figures are not independently verified by us.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 68.8888888888889,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 39.99999999999999
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 68.8888888888889
      },
      "calibration_gen": {
       "accuracy": 0.43564356435643564,
       "cc": 12.57659508660304,
       "cc_by_type": {
        "choice": 45.91728525980912,
        "noul": -12,
        "score": 3.8124999999999964
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 12.57659508660304
      },
      "composite_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 14.499339865193514,
       "cc_by_type": {
        "choice": 28.45528455284553,
        "noul": -11.111111111111116,
        "score": 26.15384615384613
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 14.499339865193514
      },
      "fairness": {
       "accuracy": 0.6666666666666666,
       "cc": 58.666666666666664,
       "cc_by_type": {
        "choice": 100,
        "noul": 46.66666666666666,
        "score": 29.333333333333357
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 58.666666666666664
      },
      "jaggedness": {
       "accuracy": 0.3,
       "cc": -17.725665313164612,
       "cc_by_type": {
        "choice": 14.82194417709337,
        "noul": -67.56756756756756,
        "score": -0.4313725490196374
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.6773162939297125,
       "cc": 51.702956719851535,
       "cc_by_type": {
        "choice": 53.24469490182266,
        "noul": 54.639175257731964,
        "score": 47.224999999999994
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 51.702956719851535
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -2.345442644411713,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -25,
        "score": -7.8095238095237995
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5781818181818181,
       "cc": 29.856090351477416,
       "cc_by_type": {
        "choice": 55.0048907727421,
        "noul": 0,
        "score": 34.56338028169014
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 29.856090351477416
      },
      "negation_pair_gen": {
       "accuracy": 0.45348837209302323,
       "cc": 1.1666666666666685,
       "cc_by_type": {
        "choice": 29.999999999999996,
        "noul": -19.999999999999996,
        "score": -6.499999999999995
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 1.1666666666666685
      },
      "option_order": {
       "accuracy": 0.4583333333333333,
       "cc": 30.758988015978694,
       "cc_by_type": {
        "choice": 30.758988015978694
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 30.758988015978694
      },
      "perturb_pair": {
       "accuracy": 0.703125,
       "cc": 51.24339724339725,
       "cc_by_type": {
        "choice": 60.96096096096097,
        "noul": 39.99999999999999,
        "score": 52.76923076923077
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 51.24339724339725
      },
      "usecase": {
       "accuracy": 0.45454545454545453,
       "cc": 10.922483660130728,
       "cc_by_type": {
        "choice": 43.627450980392176,
        "noul": -37.5,
        "score": 26.640000000000008
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 10.922483660130728
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -7.428571428571431,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": 0,
        "score": -8.000000000000007
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": -31.578947368421055,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 34.25555555555554,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 36.099999999999966
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 34.25555555555554
      },
      "de": {
       "accuracy": 0.75,
       "cc": 69.27369501090163,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 100,
        "score": 55.60869565217392
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 69.27369501090163
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 16.178150217256363,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -66.66666666666667,
        "score": 99
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 16.178150217256363
      },
      "en": {
       "accuracy": 0.49869678540399653,
       "cc": 19.641346933406236,
       "cc_by_type": {
        "choice": 39.81035135036182,
        "noul": -1.4388489208633115,
        "score": 20.552538370720196
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 19.641346933406236
      },
      "es": {
       "accuracy": 0.7096774193548387,
       "cc": 47.07481687617951,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 100,
        "score": -29.411764705882337
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 47.07481687617951
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 8.586520947176682,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 24.666666666666657
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 8.586520947176682
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 44.67801857585139,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 55.210526315789465
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 44.67801857585139
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 11.746741154562386,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 0,
        "score": -48
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 11.746741154562386
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 41.21834984968265,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": -48.285714285714306
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 41.21834984968265
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 32.59182772923837,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 19.999999999999996,
        "score": 20.66666666666668
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 32.59182772923837
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 13.632275132275131,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 0,
        "score": 29.78571428571428
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 13.632275132275131
      },
      "mixed": {
       "accuracy": 0.7142857142857143,
       "cc": 63.166666666666664,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 33.33333333333333,
        "score": 89.5
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 63.166666666666664
      },
      "nl": {
       "accuracy": 0.4,
       "cc": 13.123520923520921,
       "cc_by_type": {
        "choice": -3.8961038961038925,
        "noul": -33.333333333333336,
        "score": 76.6
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 13.123520923520921
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 48.154267830605555,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 19.697674418604638
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 48.154267830605555
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 65.99718034989006,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 40.41935483870967
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 65.99718034989006
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -10.209219858156024,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100,
        "score": 71.50000000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": 7.427536231884059,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0,
        "score": -12.5
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 7.427536231884059
      },
      "uk": {
       "accuracy": 1,
       "cc": 96.16666666666667,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 88.5
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 96.16666666666667
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 13.080979284369107,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -2.000000000000024
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 13.080979284369107
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5613333333333334,
       "cc": 43.339903174572804,
       "cc_by_type": {
        "choice": 43.339903174572804
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 43.339903174572804
      },
      "noul": {
       "accuracy": 0.512,
       "cc": 2.400000000000002,
       "cc_by_type": {
        "noul": 2.400000000000002
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 2.400000000000002
      },
      "score": {
       "accuracy": 0.448,
       "cc": 22.890794223826717,
       "cc_by_type": {
        "score": 22.890794223826717
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 22.890794223826717
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -10.70588235294118,
       "cc_by_type": {
        "choice": 36,
        "noul": -100,
        "score": 31.88235294117647
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 19.284192458105505,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -11.111111111111116,
        "score": 6.23076923076924
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 19.284192458105505
      },
      "customer_support": {
       "accuracy": 0.9285714285714286,
       "cc": 78.24661246612466,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100,
        "score": 51.00000000000002
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 78.24661246612466
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -24.921052631578956,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": 30.499999999999982
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 22.287439613526573,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": -8.499999999999996
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.287439613526573
      },
      "feature_extraction": {
       "accuracy": 0.15789473684210525,
       "cc": -142.04297693920336,
       "cc_by_type": {
        "choice": -0.62893081761006,
        "noul": -100,
        "score": -325.5
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 8.24678589384471,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -71.42857142857143,
        "score": 35.38461538461537
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 8.24678589384471
      },
      "gaming": {
       "accuracy": 0.13636363636363635,
       "cc": -29.930162096652026,
       "cc_by_type": {
        "choice": -4.529616724738673,
        "noul": -100,
        "score": 14.739130434782599
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 11.18518518518519,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 56.05555555555557
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 11.18518518518519
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": -0.7767395583241322,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": -33.333333333333336,
        "score": 8.090909090909115
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.44,
       "cc": 0.15378592122778892,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": -55.55555555555556,
        "score": -20.727272727272705
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0.15378592122778892
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -3.2831541218638036,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -33.333333333333336,
        "score": -12.00000000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 21.687388924433062,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 19.61111111111109
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 21.687388924433062
      },
      "model_routing": {
       "accuracy": 0.6912442396313364,
       "cc": 51.99623466810966,
       "cc_by_type": {
        "choice": 61.42857142857144,
        "noul": 57.57575757575757,
        "score": 36.98437499999998
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 51.99623466810966
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 31.68412931391366,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 0,
        "score": 41.11764705882352
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 31.68412931391366
      },
      "recruiting": {
       "accuracy": 0.5,
       "cc": 15.080160940885184,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 12.000000000000032
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 15.080160940885184
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -13.665217811262439,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -100,
        "score": 18.117647058823515
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 20.942995169082124,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 54.133333333333326
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 20.942995169082124
      },
      "search_retrieval": {
       "accuracy": 0.7083333333333334,
       "cc": 47.76512455516015,
       "cc_by_type": {
        "choice": 57.295373665480426,
        "noul": 0,
        "score": 86.00000000000001
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 47.76512455516015
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.59338177135288,
      80.56583679326516
     ],
     "I": [
      22.166108712128455,
      31.06134159919504
     ],
     "capability": [
      47.643755360051685,
      55.018609381039205
     ],
     "composite": null,
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.47576560172445,
     "capability": 52.14292391247805,
     "intelligence": 26.810082223231646
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5613333333333334,
      "calibration": 76.72131147540982,
      "cc": 43.339903174572804,
      "cc_by_type": {
       "choice": 43.339903174572804
      },
      "cc_tiered": 44.08558695643654,
      "cc_tiered_open": 47.260855396922054,
      "cc_tiered_sealed": 40.910318515951026,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 43.339903174572804
     },
     "noul": {
      "accuracy": 0.512,
      "calibration": 75.53887399463808,
      "cc": 2.400000000000002,
      "cc_by_type": {
       "noul": 2.400000000000002
      },
      "cc_tiered": 9.348861283643894,
      "cc_tiered_open": 18.697722567287787,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 2.400000000000002
     },
     "score": {
      "accuracy": 0.448,
      "calibration": 80.16711133512545,
      "cc": 22.890794223826717,
      "cc_by_type": {
       "score": 22.890794223826717
      },
      "cc_tiered": 26.995798429614503,
      "cc_tiered_open": 35.211116817896475,
      "cc_tiered_sealed": 18.78048004133253,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 22.890794223826717
     }
    },
    "raw_on_A": {
     "calibration": 79.09742907123326,
     "capability": 53.09845110814928,
     "intelligence": 27.099473145065282
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.47471048119585,
    "Emphasis on Accuracy 60:20:20": 10.713035887519924,
    "Emphasis on Cost 20:20:60": 17.395919380313487,
    "Emphasis on Speed 20:60:20": 17.721364339470345,
    "Intelligence only": 7.70822581673003
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bynn Intelligence (Hugging Face)",
   "axes": {
    "calibration": 77.66777221653875,
    "cost": 86.77310580145102,
    "intelligence": 17.55610652928715,
    "speed": 92.90461510807665
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10416393442622954,
      "ece_score": 79.1672131147541,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 79.1672131147541,
      "score_before_time_limit": 79.1672131147541,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.5308310991957105,
      "brier": 0.21568900804289548,
      "ece": 0.10321715817694371,
      "n": 373,
      "score": 79.35656836461126,
      "score_before_time_limit": 79.35656836461126,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8844086021505376,
      "interval90_width": 0.5495071684587812,
      "n": 372,
      "nrps": 0.15062435035842298,
      "score": 74.4795351702509,
      "score_before_time_limit": 74.4795351702509,
      "time_limit_failed_share": 0,
      "top_ece": 0.17989247311827955
     }
    },
    "score": 77.66777221653875
   },
   "capability": 47.61193937291295,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.8B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 677,924 input + 0 output tokens over 1,485 decisions = 456.514478114 tokens/decision. GPU-time cost USD 0.00275991987066432/1,000; old list-price cost USD 0.00456514478114478/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00275991987066432/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0045194933333333335
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0027599198706643154
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "DecisionNode Edge Mini 0.9.2",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.574/h as recorded by the runner (300 GB disk configured). One VM for both rows back to back with verified node shredding after each row and a final shred. The author's server (127.0.0.1:8000) in a --network none --read-only container per row; driver requests serial over loopback. Read-only code, model and input, empty credential variables, HF offline flags; cut proof inside the container before any sealed input (only loopback, no default route, DNS and TCP fail). Measured 24 Oct 2026 (reservation 05:33:57Z, VM created 05:35:01Z, teardown confirmed 05:43:05Z); this row's measurement container 91 s (run cap 109 s), scored loop 49.297 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 46b320c66dc0821d94fcd4a60a58a837d06f331357002b244e7e6c7fa4176cb3. Node-shred receipt verified for each row; exact-id VM and volume teardown confirms provider absent and guard closed. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 20.5876783664135,
    "I_sealed": 14.524534692160799,
    "base": 17.55610652928715,
    "excess": -3.776383096103011,
    "gap": 6.0631436742527,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 36.313916797579814,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 57.189189189189186,
       "hard": 32.05410506448569,
       "judge": 32.76203032300594,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -30.434782608695656,
       "judge": -30.000000000000004,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 25.44911830166068,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 35.272727272727266,
       "hard": 15.601851851851867,
       "judge": 30.81355932203388,
       "standard": 32.1851851851852
      }
     },
     "sealed|choice": {
      "cc": 34.46999359481248,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 54.2566709021601,
       "hard": 28.703596623106264,
       "judge": 33.00685602350637,
       "standard": 38.30415524151024
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -24.13793103448276,
       "hard": -34.54545454545455,
       "judge": -32.30769230769231,
       "standard": -22.388059701492537
      }
     },
     "sealed|score": {
      "cc": 9.103610481669918,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 8.447368421052648,
       "hard": 4.9887323943662025,
       "judge": 11.671874999999975,
       "standard": 13.809090909090893
      }
     }
    }
   },
   "jevbench_score": 5.353100170436692,
   "key": "decisionnode-edge-mini-0.9.2",
   "last_measured_on": "2026-10-24",
   "licence": "AGPL-3.0-only for the weights and the decisionnode_edge runtime (LICENSING.md, LICENSE, manifest licence agpl-3.0; commercial licence offered separately by Bynn Intelligence Inc.); base model Qwen/Qwen3.5-0.8B-Base Apache-2.0 (LICENSE-base); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "bynn-intelligence-com/decisionnode-edge-mini@b1ca52601cd95ef543bbe2bd0cebae2ed3f3b3fa",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.7885714285714286,
    "decisive": 175,
    "decisive_rate": 0.4691689008042895,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 227,
   "ranked": true,
   "ranks": {
    "A": 227,
    "B": 227,
    "C": 227,
    "capability": 221
   },
   "repo": "https://huggingface.co/bynn-intelligence-com/decisionnode-edge-mini",
   "scores": {
    "A": 5.353100170436692,
    "B": 4.13481200811726,
    "C": 3.71743067391437
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21277779499998814,
    "p50_s_raw": 0.03138889749999407,
    "p95_s_adjusted": 0.2407754400000101,
    "p95_s_raw": 0.04538772000000506
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run unmodified: bynn-intelligence-com/decisionnode-edge-mini at b1ca52601cd95ef543bbe2bd0cebae2ed3f3b3fa (tag v0.9.2) with the authors' PyPI package decisionnode-edge 0.9.2 (wheel sha256 dfb0b48c1233c32a106d8eee4d6c320dc3454026a1119851787f345b2f5c6317, sdist sha256 2f1da8de0be9edb3a58c3804dc024cede75b53dcea9aa82a42645e1a189ccbd0): 11 Python modules, installed from the hash-pinned wheel with --no-deps --no-index under a network-free image build; the 11 modules equal (git blob oid) runtime/src/decisionnode_edge in both Hub repos at their pins. Only the 10 data files of the pinned revision were mounted read-only at /model (config.json, decisionnode_edge.json, calibration.json, box_calibration.json, tokenizer.json, tokenizer_config.json, preprocessor_config.json, video_preprocessor_config.json, model.safetensors, readout.safetensors); no Python, GGUF or shell file of the Hub repo was fetched. Command: env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 USE_HUB_KERNELS=NO DECISIONNODE_EDGE_BACKEND=torch PYTHONDONTWRITEBYTECODE=1 python3 -m decisionnode_edge serve /model --host 127.0.0.1 --port 8000 --backend torch. PyTorch bf16 reference backend, one request at a time, no optional kernels (flash-linear-attention, causal-conv1d, Hub kernels, openvino and mlx absent; checked in the image). Image: vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 used as the CUDA 13.0 / PyTorch base (vLLM 0.30.0 is installed in it but was not used: the author's default --backend auto would select vllm there, so torch was set on the command line and in the environment, and /healthz reported backend torch after the sealed run), derived image sha256:b1f31eab61d184033ae78a92aa778c7faa2c909f8566725b4716d7803535dd47 built offline from three hash-pinned wheels (decisionnode_edge 0.9.2, transformers 5.18.0 sha256 d79e5a515a572ee3deb33eb0d578be47d91e70c75d54cf21bf6eede6edef14a9, huggingface_hub 1.33.0 sha256 04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899). Python 3.12.3, torch 2.13.0+cu130, transformers 5.18.0, tokenizers 0.23.2, safetensors 0.8.0, numpy 2.2.6. Code gate: cleared by the benchmark owner on the disclosed patterns. DISCLOSED PATTERNS: dormant subprocess start of a llama-server child in the llamacpp backend (backends.py:353), reachable only with --backend llamacpp; GGUF build subprocesses (build.py:134,136), a file imported by nothing and run only as python -m decisionnode_edge.build; Hub-capable from_pretrained loaders (model.py:49,92,235) that receive the local hash-checked model folder, under HF_HUB_OFFLINE=1, TRANSFORMERS_OFFLINE=1 and a container without network. No runtime download, trust_remote_code, string eval/exec, pickle, telemetry or request-body logging in the author code. The source and runner were reviewed before the run, with the required conditions enforced. A leftover .cache directory that would have stopped the run before any sealed item was removed and the fix rechecked; the exact device selection change was also reviewed. Weights: AGPL-3.0, public, ungated, 853,051,456 bf16 parameters, 10 data files of 1,727,281,260 bytes in total hash-checked file by file (verdict OK); prereg manifest sha256 4c7373fe2e2f94ef2f0df038708580dd2acbdae840f595142155c2cae1e27abb. The Hub main branch of the repo is one commit ahead of the pinned tag (0e77dfdc), with zero file differences against the pinned tag commit (added 0, removed 0, changed 0 in our file-by-file comparison); the measurement stays at the pinned commit b1ca5260. FIRST MEASUREMENT of DecisionNode Edge Mini 0.9.2 (our own intake; filed as GitHub issue 231 by the account @bynnit on 23 Oct 2026). No DecisionNode row exists on the live JevBench board (v1.6.42, checked 24 Oct 2026). AUTHOR'S OWN SERVER on our rented hardware, PyTorch bf16 reference backend, one request at a time. NATIVE OUTPUTS: probabilities over the item's options from the model's readout head with the author's calibration applied by the server (choice argmax and probabilities; score level, expectation and probabilities; noul P(true) from the server's truth answer reported as a noul); the server rounds probabilities to 2 decimals; nothing is generated. NAMING NOTE: the package, the PyPI release and the Hub tags are 0.9.2, but the served model name reads decisionnode-edge-mini-0.9.1 (/healthz, /v1/models and every response): the name string comes from decisionnode_edge.json in the model folder. the source review of 23 Oct 2026 found model.safetensors, readout.safetensors, config.json, tokenizer.json and calibration.json byte-identical across the tags v0.9.1 and v0.9.2; v0.9.2 adds the /v1/systemone server, box answers and documentation. The measured model tensors are therefore the 0.9.1 weights, served by the 0.9.2 runtime. PYTORCH PATH: the author's default backend selection would have used vLLM in this image; we forced the PyTorch reference path, without the optional flash-linear-attention and causal-conv1d kernels. The author's README says each backend agrees with the PyTorch reference on at least 98% of top answers and that the PyTorch path without the kernels gives the same answers about twice as slowly on long inputs; we did not test either statement. The README's timing (7.3 ms at about 500 tokens, vLLM backend, RTX PRO 6000) does not transfer to this PyTorch measurement. AUTHOR EXPOSURE (volunteered by the author in the submission and recorded in the source review; not verified by us): no JevBench item is in the training data (every training record was checked against the public items by exact text and overlapping word windows, and matches were dropped), but the public items were one of the authors' development suites: they read scores on them while comparing training runs, which is model selection against the public split. No overlap or contamination audit was run by us. LICENCE NOTE: AGPL-3.0 for the weights and the runtime; Bynn Intelligence Inc. also offers a separate commercial licence (LICENSING.md); the licence-attribution requirement has not been resolved. NOT ANSWERED, counted wrong: 15 items exceed the author's 16,384-token context limit (HTTP 400, error_type max_tokens_exceeded), exactly the 15 standard very long items (76,309-80,466 tokens by our first-party offline count) at positions in the pre-registered long-item set, as pre-registered by id and position; never truncated. Answered: 1,485 of the 1,500 items sent, HTTP 200 (status counts 200: 1,485, 400: 15); the longest answered item was 1,794 tokens. 10 s RULE: 0 decisions over 10 s, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); maximum 0.108 s over the 1,500 requests (median 0.0314 s, p95 0.0454 s). DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0 (compared on the server's 2-decimal outputs). CONTEXT: author-imposed limit 16,384 tokens; no truncation. Pre-sealed synthetic probes (not benchmark items): a 16,384-token request was accepted (0.64 s), requests of 16,385 and 82,000 tokens were refused with HTTP 400 and the server stayed healthy. The vendor's reported figures are not independently verified by us.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 59.555555555555564,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 12.00000000000001
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 59.555555555555564
      },
      "calibration_gen": {
       "accuracy": 0.3564356435643564,
       "cc": -6.900660569105679,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -52,
        "score": 4.468750000000032
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.40594059405940597,
       "cc": -3.3694670280036214,
       "cc_by_type": {
        "choice": 15.447154471544714,
        "noul": -55.55555555555556,
        "score": 29.999999999999982
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.5625,
       "cc": 19.568783068783066,
       "cc_by_type": {
        "choice": 71.42857142857143,
        "noul": -19.999999999999996,
        "score": 7.2777777777777715
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 19.568783068783066
      },
      "jaggedness": {
       "accuracy": 0.3,
       "cc": -21.841324368018586,
       "cc_by_type": {
        "choice": 16.265640038498567,
        "noul": -62.16216216216216,
        "score": -19.627450980392158
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.5143769968051118,
       "cc": 24.437463548299828,
       "cc_by_type": {
        "choice": 40.56529012943559,
        "noul": 9.278350515463906,
        "score": 23.468749999999982
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 24.437463548299828
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -12.437407952871878,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -50,
        "score": -0.714285714285734
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.49818181818181817,
       "cc": 17.71242443369172,
       "cc_by_type": {
        "choice": 45.2233452885556,
        "noul": -13.888888888888884,
        "score": 21.80281690140844
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 17.71242443369172
      },
      "negation_pair_gen": {
       "accuracy": 0.3372093023255814,
       "cc": -13.444444444444448,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -40,
        "score": -17.000000000000014
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.4166666666666667,
       "cc": 25.43275632490014,
       "cc_by_type": {
        "choice": 25.43275632490014
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 25.43275632490014
      },
      "perturb_pair": {
       "accuracy": 0.640625,
       "cc": 37.113190113190115,
       "cc_by_type": {
        "choice": 54.95495495495495,
        "noul": 0,
        "score": 56.38461538461539
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 37.113190113190115
      },
      "usecase": {
       "accuracy": 0.42272727272727273,
       "cc": 2.1034771241830064,
       "cc_by_type": {
        "choice": 46.07843137254903,
        "noul": -62.5,
        "score": 22.731999999999985
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 2.1034771241830064
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": 3.0134310134310094,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0,
        "score": -14.769230769230779
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 3.0134310134310094
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 23.664004096262158,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": 45.800000000000004
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 23.664004096262158
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 46.290881108118505,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 50,
        "score": 28.695652173913043
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 46.290881108118505
      },
      "el": {
       "accuracy": 0.4,
       "cc": 22.098075729360644,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 100
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 22.098075729360644
      },
      "en": {
       "accuracy": 0.42224152910512597,
       "cc": 3.098790707980498,
       "cc_by_type": {
        "choice": 32.121892581247444,
        "noul": -33.09352517985612,
        "score": 10.268004722550172
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 3.098790707980498
      },
      "es": {
       "accuracy": 0.5806451612903226,
       "cc": 37.726001983175,
       "cc_by_type": {
        "choice": 51.06035889070147,
        "noul": 75,
        "score": -12.882352941176457
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 37.726001983175
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -13.009107468123856,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": 59.33333333333335
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 30.139318885448915,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0,
        "score": 53.94736842105263
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 30.139318885448915
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": -1.798184357541892,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25,
        "score": -46.87499999999998
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 41.12311175444456,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": -48.57142857142858
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 41.12311175444456
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 18.537251787132636,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 0,
        "score": 12.799999999999978
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 18.537251787132636
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -10.219576719576722,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 13.78571428571428
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.7142857142857143,
       "cc": 60.166666666666664,
       "cc_by_type": {
        "choice": 50,
        "noul": 33.33333333333333,
        "score": 97.16666666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 60.166666666666664
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -2.502453102453105,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": -48.20000000000002
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 45.10349476916414,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 39.99999999999999,
        "score": 18.162790697674414
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 45.10349476916414
      },
      "pt": {
       "accuracy": 0.5714285714285714,
       "cc": 30.429271136287316,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 19.999999999999996,
        "score": 24.322580645161295
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 30.429271136287316
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -26.542553191489365,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100,
        "score": 22.499999999999986
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.23076923076923078,
       "cc": -23.398550724637683,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": 0,
        "score": -61.5
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 11.833333333333334,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 35.499999999999986
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 11.833333333333334
      },
      "zh": {
       "accuracy": 0.2,
       "cc": -51.34274952919021,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": -33.333333333333336,
        "score": -119
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5013333333333333,
       "cc": 35.59004190665723,
       "cc_by_type": {
        "choice": 35.59004190665723
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 35.59004190665723
      },
      "noul": {
       "accuracy": 0.368,
       "cc": -26.400000000000002,
       "cc_by_type": {
        "noul": -26.400000000000002
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.3973333333333333,
       "cc": 12.228339350180606,
       "cc_by_type": {
        "score": 12.228339350180606
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 12.228339350180606
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -7.941176470588242,
       "cc_by_type": {
        "choice": 52,
        "noul": -100,
        "score": 24.176470588235276
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": -11.905620144750577,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -77.77777777777779,
        "score": -2.0384615384615223
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 55.40650406504064,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 64.99999999999999
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 55.40650406504064
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -32.70711500974658,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -16.249999999999986
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": 15.20652173913043,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": 33.33333333333333,
        "score": -15.250000000000007
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 15.20652173913043
      },
      "feature_extraction": {
       "accuracy": 0.21052631578947367,
       "cc": -88.01677148846962,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100,
        "score": -176.00000000000003
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": 1.9702650290885515,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -71.42857142857143,
        "score": 26.35897435897434
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 1.9702650290885515
      },
      "gaming": {
       "accuracy": 0.22727272727272727,
       "cc": -19.860071706307128,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100,
        "score": 24.043478260869545
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.2,
       "cc": -15.777777777777771,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -100,
        "score": 56.83333333333336
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.09090909090909091,
       "cc": -32.09457530335474,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -100,
        "score": 6.500000000000017
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": -1.0689844178216248,
       "cc_by_type": {
        "choice": 65.11627906976744,
        "noul": -77.77777777777779,
        "score": 9.45454545454546
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -15.502688172043015,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100,
        "score": -14.250000000000007
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.4444444444444444,
       "cc": -5.779395038032277,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 5.694444444444413
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.5898617511520737,
       "cc": 32.08239763708514,
       "cc_by_type": {
        "choice": 60.259740259740276,
        "noul": 15.15151515151516,
        "score": 20.83593749999998
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 32.08239763708514
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": -7.629596176282415,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 23.176470588235286
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.5,
       "cc": 38.33391662500298,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 33.33333333333333,
        "score": 31.807692307692324
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 38.33391662500298
      },
      "risk_assessment": {
       "accuracy": 0.2222222222222222,
       "cc": -23.473679126823143,
       "cc_by_type": {
        "choice": 26.10837438423645,
        "noul": -100,
        "score": 3.4705882352941253
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": 12.323188405797097,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 42.76666666666665
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 12.323188405797097
      },
      "search_retrieval": {
       "accuracy": 0.5833333333333334,
       "cc": 34.23003683586189,
       "cc_by_type": {
        "choice": 57.295373665480426,
        "noul": 0,
        "score": 45.39473684210525
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 34.23003683586189
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.59220018450009,
      79.71111359708684
     ],
     "I": [
      13.478279881705168,
      20.593532034549767
     ],
     "capability": [
      43.4654134135744,
      49.54844595245572
     ],
     "composite": null,
     "seed": 30
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.66777221653875,
     "capability": 47.61193937291295,
     "intelligence": 17.55610652928715
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5013333333333333,
      "calibration": 79.1672131147541,
      "cc": 35.59004190665723,
      "cc_by_type": {
       "choice": 35.59004190665723
      },
      "cc_tiered": 35.39195519619615,
      "cc_tiered_open": 36.313916797579814,
      "cc_tiered_sealed": 34.46999359481248,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 35.59004190665723
     },
     "noul": {
      "accuracy": 0.368,
      "calibration": 79.35656836461126,
      "cc": -26.400000000000002,
      "cc_by_type": {
       "noul": -26.400000000000002
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.3973333333333333,
      "calibration": 74.4795351702509,
      "cc": 12.228339350180606,
      "cc_by_type": {
       "score": 12.228339350180606
      },
      "cc_tiered": 17.276364391665297,
      "cc_tiered_open": 25.44911830166068,
      "cc_tiered_sealed": 9.103610481669918,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 12.228339350180606
     }
    },
    "raw_on_A": {
     "calibration": 76.68973300664038,
     "capability": 47.40647606143968,
     "intelligence": 18.123219116238985
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 4.667108000793867,
    "Emphasis on Accuracy 60:20:20": 3.1911670831464813,
    "Emphasis on Cost 20:20:60": 6.025928598505021,
    "Emphasis on Speed 20:60:20": 6.116887035282391,
    "Intelligence only": 2.164435326956283
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Bynn",
   "axes": {
    "calibration": 88.59704627507568,
    "cost": null,
    "intelligence": 48.061792037330996,
    "speed": 90.91186417999961
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07117142857142862,
      "ece_score": 85.76571428571428,
      "mean_tvd": null,
      "n": 746,
      "n_ece_hard": 350,
      "n_tvd": 0,
      "score": 85.76571428571428,
      "score_before_time_limit": 85.76571428571428,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.267379679144385,
      "brier": 0.0898625668449198,
      "ece": 0.04743315508021396,
      "n": 374,
      "score": 90.5133689839572,
      "score_before_time_limit": 90.5133689839572,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9253333333333333,
      "interval90_width": 0.33155555555555555,
      "n": 375,
      "nrps": 0.07573222222222224,
      "score": 89.51205555555555,
      "score_before_time_limit": 89.51205555555555,
      "time_limit_failed_share": 0,
      "top_ece": 0.0670133333333334
     }
    },
    "score": 88.59704627507568
   },
   "capability": 68.32941915620334,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "NOT PRICED. The vendor states USD 0.042 (DecisionNode) / 0.035 (Flash) per 1M input tokens, output not billed, but under the frozen pricing rules a hosted model's own tariff counts only once it is admitted to the frozen list-price set, and these models declare no base model. Token usage on the fixed 1,489-item cost set: 471,195 input, 0 output (473,406 input on all 1,495 answered items). The vendor tariff is not used to compute a cost.",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "Hosted API by Bynn, measured directly through the vendor's synchronous endpoint. All 1,500 items were sent; 5 items over the provider's input limit were refused (HTTP 400 max_tokens_exceeded), scored as misses and kept in the denominator. The provider reports probabilities to two decimals; exact zeros on many-option items are expected, and Calibration is computed on the reported values without adjustment. Output tokens reported as 0. Measured on the 27 Oct 2026 regular draw, not yet equated to the reference scale (offset 0). This draw has no subject-topic labels yet; use-case and language cells are measured on its own draw. Latency: p50 0.256 s, p95 0.317 s, maximum 0.668 s.",
   "display": "DecisionNode Flash",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-28",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.19",
    "results_sha256": "9bf869f47a08a0ebe1b00f6fbc1643b681a68554f95221b3424670c15bfaa1c6",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Hosted API by Bynn; direct synchronous vendor endpoint https://api.decisionnode.com/v1/systemone.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 50.98015834464718,
    "I_sealed": 45.14342573001481,
    "base": 48.061792037330996,
    "excess": 3.263344972194124,
    "gap": 5.836732614632368,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 68.97125530746317,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 68.35860250494397,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 42.733609385783296,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 47.82608695652173,
       "judge": 30.000000000000004,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 41.23561034069509,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 37.696969696969695,
       "hard": 43.4537037037037,
       "judge": 40.62711864406781,
       "standard": 39.481481481481474
      }
     },
     "sealed|choice": {
      "cc": 60.03826154245532,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 84.27775958074025,
       "hard": 45.806042679061896,
       "judge": 64.79723899913719,
       "standard": 69.24448406507692
      }
     },
     "sealed|noul": {
      "cc": 28.234761802309283,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 72.41379310344827,
       "hard": -2.4390243902439046,
       "judge": 42.22222222222223,
       "standard": 46.51162790697674
      }
     },
     "sealed|score": {
      "cc": 47.15725384527985,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 81.9890909090909,
       "hard": 22.432432432432424,
       "judge": 57.03409090909091,
       "standard": 64.37572254335258
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "decisionnode-flash-0.9.1-rc1",
   "last_measured_on": "2026-10-28",
   "licence": "proprietary hosted API",
   "listing": "unranked",
   "measured_in": "v1.7.19",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": {
    "model": "decisionnode-flash-0.9.1-rc1",
    "served_model": "decisionnode-flash-0.9.1-rc1"
   },
   "new_in": "v1.7.19",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale; stronger anchor runs for that draw are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.9635036496350365,
    "decisive": 274,
    "decisive_rate": 0.732620320855615,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "provider": "bynn",
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://decisionnode.com",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.256085725966841,
    "p50_s_raw": 0.256085725966841,
    "p95_s_adjusted": 0.31653971727064345,
    "p95_s_raw": 0.31653971727064345
   },
   "status": {
    "answered_ok": 1495,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.75,
       "cc": 66.66666666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 66.66666666666666
      },
      "calibration_gen": {
       "accuracy": 0.2857142857142857,
       "cc": 21.248196248196244,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": 100,
        "score": -58.33333333333335
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 21.248196248196244
      },
      "composite_gen": {
       "accuracy": 0.631578947368421,
       "cc": 35.27777777777778,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 19.999999999999996,
        "score": 12.49999999999999
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 35.27777777777778
      },
      "fairness": {
       "accuracy": 1,
       "cc": 97.94444444444444,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 93.83333333333333
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 97.94444444444444
      },
      "jaggedness": {
       "accuracy": 0.5909090909090909,
       "cc": 11.353909465020577,
       "cc_by_type": {
        "choice": 50.617283950617285,
        "noul": -33.333333333333336,
        "score": 16.77777777777778
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 11.353909465020577
      },
      "llm_router": {
       "accuracy": 0.7777777777777778,
       "cc": 69.52397372425884,
       "cc_by_type": {
        "choice": 73.56051703877792,
        "noul": 73.91304347826086,
        "score": 61.09836065573771
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 69.52397372425884
      },
      "long_state": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0,
       "cc": -66.66666666666667,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.7231270358306189,
       "cc": 53.805512669275025,
       "cc_by_type": {
        "choice": 62.49191636128479,
        "noul": 40.98360655737705,
        "score": 57.94101508916323
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 53.805512669275025
      },
      "negation_pair_gen": {
       "accuracy": 0.6363636363636364,
       "cc": 35,
       "cc_by_type": {
        "choice": 86.66666666666667,
        "noul": -16.666666666666664
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 35
      },
      "option_order": {
       "accuracy": 0.7037037037037037,
       "cc": 61.90476190476191,
       "cc_by_type": {
        "choice": 61.90476190476191
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 61.90476190476191
      },
      "perturb_pair": {
       "accuracy": 0.8928571428571429,
       "cc": 78.34019439679817,
       "cc_by_type": {
        "choice": 90.56603773584905,
        "noul": 80,
        "score": 64.45454545454545
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 78.34019439679817
      },
      "usecase": {
       "accuracy": 0.6615384615384615,
       "cc": 45.391999557207434,
       "cc_by_type": {
        "choice": 54.04814004376365,
        "noul": 35.384615384615394,
        "score": 46.74324324324325
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 45.391999557207434
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.7142857142857143,
       "cc": 58.06195720297251,
       "cc_by_type": {
        "choice": 45.20547945205479,
        "noul": 33.33333333333333,
        "score": 95.6470588235294
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 58.06195720297251
      },
      "cs": {
       "accuracy": 0.7027027027027027,
       "cc": 48.274905921964745,
       "cc_by_type": {
        "choice": 75.93582887700535,
        "noul": 11.111111111111116,
        "score": 57.77777777777777
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 48.274905921964745
      },
      "da": {
       "accuracy": 0.7222222222222222,
       "cc": 38.06173536608318,
       "cc_by_type": {
        "choice": 77.63975155279502,
        "noul": 9.090909090909083,
        "score": 27.454545454545443
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 38.06173536608318
      },
      "de": {
       "accuracy": 0.75,
       "cc": 55.018825810631824,
       "cc_by_type": {
        "choice": 67.58425520967327,
        "noul": 50,
        "score": 47.47222222222221
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 55.018825810631824
      },
      "el": {
       "accuracy": 0.4642857142857143,
       "cc": 11.489230769230767,
       "cc_by_type": {
        "choice": 42.307692307692314,
        "noul": -40,
        "score": 32.15999999999999
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 11.489230769230767
      },
      "en": {
       "accuracy": 0.6828828828828829,
       "cc": 47.29675622182828,
       "cc_by_type": {
        "choice": 60.466004583651625,
        "noul": 37.704918032786885,
        "score": 43.71934604904636
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 47.29675622182828
      },
      "es": {
       "accuracy": 0.7837837837837838,
       "cc": 69.45225578425256,
       "cc_by_type": {
        "choice": 71.13071371291097,
        "noul": 77.77777777777777,
        "score": 59.448275862068954
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 69.45225578425256
      },
      "fi": {
       "accuracy": 0.7115384615384616,
       "cc": 59.71848098507138,
       "cc_by_type": {
        "choice": 45.14285714285715,
        "noul": 78.94736842105263,
        "score": 55.065217391304344
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 59.71848098507138
      },
      "fr": {
       "accuracy": 0.6875,
       "cc": 53.37772280629423,
       "cc_by_type": {
        "choice": 59.18367346938776,
        "noul": 55.55555555555556,
        "score": 45.393939393939384
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 53.37772280629423
      },
      "hi": {
       "accuracy": 0.7567567567567568,
       "cc": 55.694474388150276,
       "cc_by_type": {
        "choice": 75.20661157024793,
        "noul": 33.33333333333333,
        "score": 58.543478260869556
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 55.694474388150276
      },
      "id": {
       "accuracy": 0.7804878048780488,
       "cc": 64.97235023041475,
       "cc_by_type": {
        "choice": 53.91705069124423,
        "noul": 66.66666666666667,
        "score": 74.33333333333333
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 64.97235023041475
      },
      "it": {
       "accuracy": 0.7555555555555555,
       "cc": 61.94485523678069,
       "cc_by_type": {
        "choice": 68.05678793256432,
        "noul": 66.66666666666667,
        "score": 51.11111111111111
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 61.94485523678069
      },
      "ja": {
       "accuracy": 0.6818181818181818,
       "cc": 44.85936764013189,
       "cc_by_type": {
        "choice": 53.36787564766839,
        "noul": 27.27272727272727,
        "score": 53.93750000000001
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 44.85936764013189
      },
      "ko": {
       "accuracy": 0.627906976744186,
       "cc": 41.33589087809037,
       "cc_by_type": {
        "choice": 48.5294117647059,
        "noul": 19.999999999999996,
        "score": 55.47826086956521
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 41.33589087809037
      },
      "mixed": {
       "accuracy": 0.875,
       "cc": 76.18243324332434,
       "cc_by_type": {
        "choice": 72.997299729973,
        "noul": 80,
        "score": 75.55
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 76.18243324332434
      },
      "nl": {
       "accuracy": 0.75,
       "cc": 45.15786163522012,
       "cc_by_type": {
        "choice": 78.77358490566037,
        "noul": 0,
        "score": 56.699999999999996
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 45.15786163522012
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 45.52724179927731,
       "cc_by_type": {
        "choice": 38.03786574870913,
        "noul": 73.33333333333334,
        "score": 25.210526315789483
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 45.52724179927731
      },
      "pl": {
       "accuracy": 0.6491228070175439,
       "cc": 40.874901537049745,
       "cc_by_type": {
        "choice": 61.681745609366686,
        "noul": 5.882352941176472,
        "score": 55.060606060606055
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 40.874901537049745
      },
      "pt": {
       "accuracy": 0.7659574468085106,
       "cc": 58.63652197604383,
       "cc_by_type": {
        "choice": 70.04991680532446,
        "noul": 33.33333333333333,
        "score": 72.52631578947368
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 58.63652197604383
      },
      "sv": {
       "accuracy": 0.6304347826086957,
       "cc": 42.5673494723774,
       "cc_by_type": {
        "choice": 46.36871508379889,
        "noul": 33.33333333333333,
        "score": 47.999999999999986
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 42.5673494723774
      },
      "tr": {
       "accuracy": 0.7391304347826086,
       "cc": 54.39106381181637,
       "cc_by_type": {
        "choice": 58.950969213226905,
        "noul": 19.999999999999996,
        "score": 84.22222222222221
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 54.39106381181637
      },
      "uk": {
       "accuracy": 0.7321428571428571,
       "cc": 51.710268822741774,
       "cc_by_type": {
        "choice": 67.12328767123289,
        "noul": 71.42857142857142,
        "score": 16.578947368421037
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 51.710268822741774
      },
      "zh": {
       "accuracy": 0.8292682926829268,
       "cc": 66.36220385674932,
       "cc_by_type": {
        "choice": 75.20661157024794,
        "noul": 50,
        "score": 73.88
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 66.36220385674932
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7173333333333334,
       "cc": 61.443251053338834,
       "cc_by_type": {
        "choice": 61.443251053338834
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 61.443251053338834
      },
      "noul": {
       "accuracy": 0.704,
       "cc": 40.79999999999999,
       "cc_by_type": {
        "noul": 40.79999999999999
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 40.79999999999999
      },
      "score": {
       "accuracy": 0.6933333333333334,
       "cc": 53.09189189189194,
       "cc_by_type": {
        "score": 53.09189189189194
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 53.09189189189194
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5416666666666666,
       "cc": 13.899519926577,
       "cc_by_type": {
        "choice": 46.627131208302444,
        "noul": -71.42857142857143,
        "score": 66.49999999999999
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 13.899519926577
      },
      "code_linting": {
       "accuracy": 0.6521739130434783,
       "cc": 45.1251669692705,
       "cc_by_type": {
        "choice": 52.2351500306185,
        "noul": 66.66666666666667,
        "score": 16.473684210526308
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 45.1251669692705
      },
      "customer_support": {
       "accuracy": 0.7575757575757576,
       "cc": 48.67353963013313,
       "cc_by_type": {
        "choice": 74.16974169741697,
        "noul": 6.666666666666665,
        "score": 65.18421052631578
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 48.67353963013313
      },
      "demand_forecasting": {
       "accuracy": 0.45454545454545453,
       "cc": 3.8225910219845125,
       "cc_by_type": {
        "choice": 18.119787717968165,
        "noul": -38.46153846153846,
        "score": 31.80952380952383
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 3.8225910219845125
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6774193548387096,
       "cc": 46.12088889892425,
       "cc_by_type": {
        "choice": 57.77621393384941,
        "noul": 17.647058823529417,
        "score": 62.93939393939394
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 46.12088889892425
      },
      "feature_extraction": {
       "accuracy": 0.8472222222222222,
       "cc": 74.72262901810639,
       "cc_by_type": {
        "choice": 77.38693467336681,
        "noul": 85.71428571428572,
        "score": 61.066666666666656
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 74.72262901810639
      },
      "financial_crime": {
       "accuracy": 0.6507936507936508,
       "cc": 33.985645933014354,
       "cc_by_type": {
        "choice": 52.15311004784688,
        "noul": -9.090909090909093,
        "score": 58.894736842105274
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 33.985645933014354
      },
      "gaming": {
       "accuracy": 0.6724137931034483,
       "cc": 49.81292607348385,
       "cc_by_type": {
        "choice": 56.97211155378488,
        "noul": 33.33333333333333,
        "score": 59.13333333333335
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 49.81292607348385
      },
      "insurance_claims": {
       "accuracy": 0.6129032258064516,
       "cc": 26.57609211464839,
       "cc_by_type": {
        "choice": 46.54788418708242,
        "noul": 17.647058823529417,
        "score": 15.533333333333331
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 26.57609211464839
      },
      "knowledge_graphs": {
       "accuracy": 0.6268656716417911,
       "cc": 21.364971360473618,
       "cc_by_type": {
        "choice": 64.01799100449774,
        "noul": -23.076923076923073,
        "score": 23.15384615384618
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 21.364971360473618
      },
      "lead_generation": {
       "accuracy": 0.7733333333333333,
       "cc": 57.23805046230998,
       "cc_by_type": {
        "choice": 70.38081805359661,
        "noul": 60.00000000000001,
        "score": 41.33333333333332
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 57.23805046230998
      },
      "legal_compliance": {
       "accuracy": 0.6071428571428571,
       "cc": 45.14055760354657,
       "cc_by_type": {
        "choice": 57.87362086258776,
        "noul": 42.85714285714286,
        "score": 34.690909090909095
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 45.14055760354657
      },
      "llm_guardrails": {
       "accuracy": 0.8450704225352113,
       "cc": 76.62125741988268,
       "cc_by_type": {
        "choice": 83.70672097759675,
        "noul": 84.61538461538463,
        "score": 61.54166666666667
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 76.62125741988268
      },
      "model_routing": {
       "accuracy": 0.8108108108108109,
       "cc": 72.8283455265363,
       "cc_by_type": {
        "choice": 76.63146962091051,
        "noul": 76.47058823529412,
        "score": 65.38297872340426
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 72.8283455265363
      },
      "moderation_trust_safety": {
       "accuracy": 0.6779661016949152,
       "cc": 50.574484803028675,
       "cc_by_type": {
        "choice": 34.640522875817,
        "noul": 64.70588235294117,
        "score": 52.37704918032786
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 50.574484803028675
      },
      "other": {
       "accuracy": 0.625,
       "cc": 47.24688617585398,
       "cc_by_type": {
        "choice": 33.40732519422864,
        "noul": 66.66666666666667,
        "score": 41.66666666666665
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 47.24688617585398
      },
      "recruiting": {
       "accuracy": 0.7692307692307693,
       "cc": 62.635683729872625,
       "cc_by_type": {
        "choice": 61.06705118961787,
        "noul": 68,
        "score": 58.84
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 62.635683729872625
      },
      "risk_assessment": {
       "accuracy": 0.6458333333333334,
       "cc": 29.881289561205875,
       "cc_by_type": {
        "choice": 56.06694560669455,
        "noul": -7.692307692307687,
        "score": 41.26923076923077
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 29.881289561205875
      },
      "scientific_discovery": {
       "accuracy": 0.5952380952380952,
       "cc": 25.044308111792777,
       "cc_by_type": {
        "choice": 48.46625766871167,
        "noul": -19.999999999999996,
        "score": 46.666666666666664
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 25.044308111792777
      },
      "search_retrieval": {
       "accuracy": 0.9701492537313433,
       "cc": 92.8313492063492,
       "cc_by_type": {
        "choice": 100,
        "noul": 85.71428571428572,
        "score": 92.7797619047619
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 92.8313492063492
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.76351642024285,
      89.37670651262867
     ],
     "I": [
      42.19615351272806,
      52.48186662345012
     ],
     "capability": [
      63.80456101344181,
      70.09973171833498
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.59704627507568,
     "capability": 68.32941915620334,
     "intelligence": 48.061792037330996
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7173333333333334,
      "calibration": 85.76571428571428,
      "cc": 61.443251053338834,
      "cc_by_type": {
       "choice": 61.443251053338834
      },
      "cc_tiered": 64.50475842495925,
      "cc_tiered_open": 68.97125530746317,
      "cc_tiered_sealed": 60.03826154245532,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.443251053338834
     },
     "noul": {
      "accuracy": 0.704,
      "calibration": 90.5133689839572,
      "cc": 40.79999999999999,
      "cc_by_type": {
       "noul": 40.79999999999999
      },
      "cc_tiered": 35.48418559404629,
      "cc_tiered_open": 42.733609385783296,
      "cc_tiered_sealed": 28.234761802309283,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 40.79999999999999
     },
     "score": {
      "accuracy": 0.6933333333333334,
      "calibration": 89.51205555555555,
      "cc": 53.09189189189194,
      "cc_by_type": {
       "score": 53.09189189189194
      },
      "cc_tiered": 44.196432092987465,
      "cc_tiered_open": 41.23561034069509,
      "cc_tiered_sealed": 47.15725384527985,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 53.09189189189194
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.005333333333333301,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": 44.40786265072641
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Bynn",
   "axes": {
    "calibration": 92.47719240500243,
    "cost": null,
    "intelligence": 61.050937711641836,
    "speed": 82.66643460386248
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.026171428571428484,
      "ece_score": 94.7657142857143,
      "mean_tvd": null,
      "n": 746,
      "n_ece_hard": 350,
      "n_tvd": 0,
      "score": 94.7657142857143,
      "score_before_time_limit": 94.7657142857143,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.17112299465240643,
      "brier": 0.0604743315508021,
      "ece": 0.05090909090909083,
      "n": 374,
      "score": 89.81818181818183,
      "score_before_time_limit": 89.81818181818183,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.952,
      "interval90_width": 0.20422222222222222,
      "n": 375,
      "nrps": 0.049926377777777795,
      "score": 92.84768111111117,
      "score_before_time_limit": 92.84768111111117,
      "time_limit_failed_share": 0,
      "top_ece": 0.046559999999999505
     }
    },
    "score": 92.47719240500243
   },
   "capability": 76.76406505832213,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "NOT PRICED. The vendor states USD 0.042 (DecisionNode) / 0.035 (Flash) per 1M input tokens, output not billed, but under the frozen pricing rules a hosted model's own tariff counts only once it is admitted to the frozen list-price set, and these models declare no base model. Token usage on the fixed 1,489-item cost set: 471,195 input, 0 output (473,406 input on all 1,495 answered items). The vendor tariff is not used to compute a cost.",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "Hosted API by Bynn, measured directly through the vendor's synchronous endpoint. All 1,500 items were sent; 5 items over the provider's input limit were refused (HTTP 400 max_tokens_exceeded), scored as misses and kept in the denominator. The provider reports probabilities to two decimals; exact zeros on many-option items are expected, and Calibration is computed on the reported values without adjustment. Output tokens reported as 0. Measured on the 27 Oct 2026 regular draw, not yet equated to the reference scale (offset 0). This draw has no subject-topic labels yet; use-case and language cells are measured on its own draw. A first attempt stopped incomplete because of a harness resend bug on our side and is not used; the published result is the second, complete attempt. Latency: p50 0.675 s, p95 0.802 s, maximum 1.408 s.",
   "display": "DecisionNode",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-28",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.19",
    "results_sha256": "9bf869f47a08a0ebe1b00f6fbc1643b681a68554f95221b3424670c15bfaa1c6",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Hosted API by Bynn; direct synchronous vendor endpoint https://api.decisionnode.com/v1/systemone.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 57.767289793708926,
    "I_sealed": 64.33458562957475,
    "base": 61.050937711641836,
    "excess": -9.140683478304064,
    "gap": -6.567295835865821,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 74.24692966214873,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 69.80182447310474,
       "judge": 68.35860250494397,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 42.46376811594203,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 47.82608695652173,
       "judge": 0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 56.59117160303602,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 64.15151515151516,
       "hard": 52.787037037037045,
       "judge": 60.796610169491515,
       "standard": 54.11111111111112
      }
     },
     "sealed|choice": {
      "cc": 71.90390367916619,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 90.17359973796265,
       "hard": 57.215296851890976,
       "judge": 78.25711820534946,
       "standard": 82.61644751504349
      }
     },
     "sealed|noul": {
      "cc": 55.58649376389514,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 83.9080459770115,
       "hard": 29.268292682926834,
       "judge": 73.33333333333334,
       "standard": 67.44186046511629
      }
     },
     "sealed|score": {
      "cc": 65.5133594456629,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 88.33454545454545,
       "hard": 48.47490347490346,
       "judge": 78.48863636363636,
       "standard": 68.71676300578034
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "decisionnode-main-0.9.1-rc1",
   "last_measured_on": "2026-10-28",
   "licence": "proprietary hosted API",
   "listing": "unranked",
   "measured_in": "v1.7.19",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": {
    "model": "decisionnode-main-0.9.1-rc1",
    "served_model": "decisionnode-main-0.9.1-rc1"
   },
   "new_in": "v1.7.19",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale; stronger anchor runs for that draw are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.967741935483871,
    "decisive": 310,
    "decisive_rate": 0.8288770053475936,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "provider": "bynn",
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://decisionnode.com",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.6751686710049398,
    "p50_s_raw": 0.6751686710049398,
    "p95_s_adjusted": 0.8015751732105854,
    "p95_s_raw": 0.8015751732105854
   },
   "status": {
    "answered_ok": 1495,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.75,
       "cc": 66.66666666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 66.66666666666666
      },
      "calibration_gen": {
       "accuracy": 0.42857142857142855,
       "cc": 38.025974025974016,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": 100,
        "score": -8.000000000000028
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 38.025974025974016
      },
      "composite_gen": {
       "accuracy": 0.7894736842105263,
       "cc": 50.666666666666664,
       "cc_by_type": {
        "choice": 100,
        "noul": -19.999999999999996,
        "score": 72
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 50.666666666666664
      },
      "fairness": {
       "accuracy": 1,
       "cc": 95.61111111111113,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 86.83333333333334
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 95.61111111111113
      },
      "jaggedness": {
       "accuracy": 0.4090909090909091,
       "cc": -14.473251028806578,
       "cc_by_type": {
        "choice": 13.580246913580252,
        "noul": -100,
        "score": 43.000000000000014
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.8444444444444444,
       "cc": 74.5403479031419,
       "cc_by_type": {
        "choice": 78.84841363102231,
        "noul": 82.6086956521739,
        "score": 62.16393442622952
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 74.5403479031419
      },
      "long_state": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.8078175895765473,
       "cc": 69.07046685333212,
       "cc_by_type": {
        "choice": 71.86893727096358,
        "noul": 63.11475409836065,
        "score": 72.22770919067212
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 69.07046685333212
      },
      "negation_pair_gen": {
       "accuracy": 0.7272727272727273,
       "cc": 51.66666666666667,
       "cc_by_type": {
        "choice": 86.66666666666667,
        "noul": 16.666666666666675
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 51.66666666666667
      },
      "option_order": {
       "accuracy": 0.8888888888888888,
       "cc": 85.71428571428571,
       "cc_by_type": {
        "choice": 85.71428571428571
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 85.71428571428571
      },
      "perturb_pair": {
       "accuracy": 0.9642857142857143,
       "cc": 83.57575757575758,
       "cc_by_type": {
        "choice": 100,
        "noul": 80,
        "score": 70.72727272727273
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 83.57575757575758
      },
      "usecase": {
       "accuracy": 0.7507692307692307,
       "cc": 63.10412600784591,
       "cc_by_type": {
        "choice": 67.1772428884026,
        "noul": 60.00000000000001,
        "score": 62.135135135135144
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 63.10412600784591
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8095238095238095,
       "cc": 72.73927835974571,
       "cc_by_type": {
        "choice": 58.904109589041084,
        "noul": 66.66666666666667,
        "score": 92.6470588235294
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 72.73927835974571
      },
      "cs": {
       "accuracy": 0.7567567567567568,
       "cc": 58.779164190928896,
       "cc_by_type": {
        "choice": 59.89304812834225,
        "noul": 33.33333333333333,
        "score": 83.1111111111111
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 58.779164190928896
      },
      "da": {
       "accuracy": 0.75,
       "cc": 41.273480143045354,
       "cc_by_type": {
        "choice": 85.09316770186335,
        "noul": 9.090909090909083,
        "score": 29.636363636363626
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 41.273480143045354
      },
      "de": {
       "accuracy": 0.8541666666666666,
       "cc": 76.07557242089014,
       "cc_by_type": {
        "choice": 81.47671726267045,
        "noul": 75,
        "score": 71.74999999999999
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 76.07557242089014
      },
      "el": {
       "accuracy": 0.75,
       "cc": 55.08102564102563,
       "cc_by_type": {
        "choice": 76.92307692307692,
        "noul": 0,
        "score": 88.32
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 55.08102564102563
      },
      "en": {
       "accuracy": 0.7603603603603604,
       "cc": 60.983669858056004,
       "cc_by_type": {
        "choice": 69.91978609625669,
        "noul": 52.45901639344262,
        "score": 60.57220708446869
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 60.983669858056004
      },
      "es": {
       "accuracy": 0.8648648648648649,
       "cc": 78.87299890106625,
       "cc_by_type": {
        "choice": 85.56535685645551,
        "noul": 77.77777777777777,
        "score": 73.27586206896551
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 78.87299890106625
      },
      "fi": {
       "accuracy": 0.8461538461538461,
       "cc": 77.40481638879808,
       "cc_by_type": {
        "choice": 72.57142857142857,
        "noul": 78.94736842105263,
        "score": 80.69565217391303
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 77.40481638879808
      },
      "fr": {
       "accuracy": 0.78125,
       "cc": 68.25719782862639,
       "cc_by_type": {
        "choice": 69.38775510204081,
        "noul": 77.77777777777777,
        "score": 57.6060606060606
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 68.25719782862639
      },
      "hi": {
       "accuracy": 0.8108108108108109,
       "cc": 70.05960128292143,
       "cc_by_type": {
        "choice": 83.47107438016529,
        "noul": 55.55555555555556,
        "score": 71.15217391304347
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 70.05960128292143
      },
      "id": {
       "accuracy": 0.8292682926829268,
       "cc": 74.19457245263696,
       "cc_by_type": {
        "choice": 53.91705069124423,
        "noul": 83.33333333333333,
        "score": 85.33333333333333
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 74.19457245263696
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 78.17613132209404,
       "cc_by_type": {
        "choice": 84.02839396628217,
        "noul": 83.33333333333333,
        "score": 67.16666666666666
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 78.17613132209404
      },
      "ja": {
       "accuracy": 0.8181818181818182,
       "cc": 68.23484357826975,
       "cc_by_type": {
        "choice": 68.91191709844558,
        "noul": 63.63636363636365,
        "score": 72.15625000000001
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 68.23484357826975
      },
      "ko": {
       "accuracy": 0.6744186046511628,
       "cc": 50.95524296675192,
       "cc_by_type": {
        "choice": 63.23529411764706,
        "noul": 19.999999999999996,
        "score": 69.6304347826087
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 50.95524296675192
      },
      "mixed": {
       "accuracy": 0.890625,
       "cc": 82.10743324332434,
       "cc_by_type": {
        "choice": 72.997299729973,
        "noul": 89.99999999999999,
        "score": 83.325
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 82.10743324332434
      },
      "nl": {
       "accuracy": 0.8055555555555556,
       "cc": 61.532704402515726,
       "cc_by_type": {
        "choice": 71.69811320754718,
        "noul": 80,
        "score": 32.900000000000006
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 61.532704402515726
      },
      "no": {
       "accuracy": 0.7647058823529411,
       "cc": 63.81332850197784,
       "cc_by_type": {
        "choice": 69.01893287435456,
        "noul": 60.00000000000001,
        "score": 62.421052631578945
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 63.81332850197784
      },
      "pl": {
       "accuracy": 0.8245614035087719,
       "cc": 69.92073349087404,
       "cc_by_type": {
        "choice": 71.26130920702501,
        "noul": 76.47058823529412,
        "score": 62.03030303030302
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 69.92073349087404
      },
      "pt": {
       "accuracy": 0.8085106382978723,
       "cc": 61.35501260083098,
       "cc_by_type": {
        "choice": 82.02995008319466,
        "noul": 16.666666666666675,
        "score": 85.36842105263159
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 61.35501260083098
      },
      "sv": {
       "accuracy": 0.8043478260869565,
       "cc": 70.48656557595105,
       "cc_by_type": {
        "choice": 79.88826815642459,
        "noul": 83.33333333333333,
        "score": 48.23809523809523
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 70.48656557595105
      },
      "tr": {
       "accuracy": 0.7391304347826086,
       "cc": 65.38833987921787,
       "cc_by_type": {
        "choice": 52.109464082098064,
        "noul": 60.00000000000001,
        "score": 84.05555555555554
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 65.38833987921787
      },
      "uk": {
       "accuracy": 0.7678571428571429,
       "cc": 60.86306519723967,
       "cc_by_type": {
        "choice": 56.16438356164384,
        "noul": 85.71428571428572,
        "score": 40.71052631578945
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 60.86306519723967
      },
      "zh": {
       "accuracy": 0.9024390243902439,
       "cc": 78.26831955922864,
       "cc_by_type": {
        "choice": 81.40495867768595,
        "noul": 75,
        "score": 78.39999999999999
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 78.26831955922864
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7893333333333333,
       "cc": 71.26430974729968,
       "cc_by_type": {
        "choice": 71.26430974729968
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 71.26430974729968
      },
      "noul": {
       "accuracy": 0.8,
       "cc": 60.00000000000001,
       "cc_by_type": {
        "noul": 60.00000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 60.00000000000001
      },
      "score": {
       "accuracy": 0.784,
       "cc": 67.94144144144141,
       "cc_by_type": {
        "score": 67.94144144144141
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 67.94144144144141
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7708333333333334,
       "cc": 71.67388630025768,
       "cc_by_type": {
        "choice": 64.41808747220162,
        "noul": 71.42857142857142,
        "score": 79.175
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 71.67388630025768
      },
      "code_linting": {
       "accuracy": 0.7971014492753623,
       "cc": 68.64823111053813,
       "cc_by_type": {
        "choice": 81.62890385793018,
        "noul": 83.33333333333333,
        "score": 40.98245614035088
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 68.64823111053813
      },
      "customer_support": {
       "accuracy": 0.7727272727272727,
       "cc": 60.080954230594934,
       "cc_by_type": {
        "choice": 70.47970479704797,
        "noul": 19.999999999999996,
        "score": 89.76315789473685
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 60.080954230594934
      },
      "demand_forecasting": {
       "accuracy": 0.42424242424242425,
       "cc": -2.7884594556315583,
       "cc_by_type": {
        "choice": 31.76648976497347,
        "noul": -53.84615384615385,
        "score": 13.714285714285701
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7741935483870968,
       "cc": 65.49507451883574,
       "cc_by_type": {
        "choice": 61.99859254046446,
        "noul": 52.941176470588225,
        "score": 81.54545454545455
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 65.49507451883574
      },
      "feature_extraction": {
       "accuracy": 0.7361111111111112,
       "cc": 57.555507697216235,
       "cc_by_type": {
        "choice": 58.54271356783919,
        "noul": 42.85714285714286,
        "score": 71.26666666666665
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 57.555507697216235
      },
      "financial_crime": {
       "accuracy": 0.7619047619047619,
       "cc": 53.18128654970761,
       "cc_by_type": {
        "choice": 74.48165869218504,
        "noul": 27.27272727272727,
        "score": 57.78947368421052
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 53.18128654970761
      },
      "gaming": {
       "accuracy": 0.7586206896551724,
       "cc": 67.15943632875904,
       "cc_by_type": {
        "choice": 66.53386454183267,
        "noul": 77.77777777777777,
        "score": 57.16666666666668
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 67.15943632875904
      },
      "insurance_claims": {
       "accuracy": 0.7096774193548387,
       "cc": 56.97347773556343,
       "cc_by_type": {
        "choice": 46.54788418708242,
        "noul": 64.70588235294117,
        "score": 59.66666666666669
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 56.97347773556343
      },
      "knowledge_graphs": {
       "accuracy": 0.8208955223880597,
       "cc": 67.06219005881674,
       "cc_by_type": {
        "choice": 73.01349325337331,
        "noul": 69.23076923076923,
        "score": 58.94230769230769
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 67.06219005881674
      },
      "lead_generation": {
       "accuracy": 0.8933333333333333,
       "cc": 82.10654398101507,
       "cc_by_type": {
        "choice": 78.84344146685473,
        "noul": 82.85714285714285,
        "score": 84.61904761904762
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 82.10654398101507
      },
      "legal_compliance": {
       "accuracy": 0.8035714285714286,
       "cc": 68.99320124876795,
       "cc_by_type": {
        "choice": 81.94583751253761,
        "noul": 57.14285714285714,
        "score": 67.89090909090909
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 68.99320124876795
      },
      "llm_guardrails": {
       "accuracy": 0.9436619718309859,
       "cc": 91.36284640625273,
       "cc_by_type": {
        "choice": 95.92668024439918,
        "noul": 92.3076923076923,
        "score": 85.85416666666667
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 91.36284640625273
      },
      "model_routing": {
       "accuracy": 0.8918918918918919,
       "cc": 79.77727431633897,
       "cc_by_type": {
        "choice": 85.97888177254633,
        "noul": 82.35294117647058,
        "score": 71.00000000000001
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 79.77727431633897
      },
      "moderation_trust_safety": {
       "accuracy": 0.8305084745762712,
       "cc": 78.29826065216615,
       "cc_by_type": {
        "choice": 73.8562091503268,
        "noul": 88.23529411764706,
        "score": 72.80327868852459
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 78.29826065216615
      },
      "other": {
       "accuracy": 0.78125,
       "cc": 76.79257614995684,
       "cc_by_type": {
        "choice": 60.04439511653718,
        "noul": 100,
        "score": 70.33333333333334
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 76.79257614995684
      },
      "recruiting": {
       "accuracy": 0.8769230769230769,
       "cc": 75.88352799807738,
       "cc_by_type": {
        "choice": 78.37058399423216,
        "noul": 100,
        "score": 49.28
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 75.88352799807738
      },
      "risk_assessment": {
       "accuracy": 0.7083333333333334,
       "cc": 49.16334084325717,
       "cc_by_type": {
        "choice": 56.06694560669455,
        "noul": 23.076923076923084,
        "score": 68.34615384615384
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 49.16334084325717
      },
      "scientific_discovery": {
       "accuracy": 0.6666666666666666,
       "cc": 40.683765053396954,
       "cc_by_type": {
        "choice": 63.190184049079754,
        "noul": 0,
        "score": 58.86111111111111
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 40.683765053396954
      },
      "search_retrieval": {
       "accuracy": 0.9701492537313433,
       "cc": 89.35119047619048,
       "cc_by_type": {
        "choice": 100,
        "noul": 71.42857142857142,
        "score": 96.625
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 89.35119047619048
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.28110104629627,
      92.43723168414547
     ],
     "I": [
      55.89072192433878,
      65.59467933295957
     ],
     "capability": [
      72.4968875690365,
      78.0476666752552
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 92.47719240500243,
     "capability": 76.76406505832213,
     "intelligence": 61.050937711641836
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7893333333333333,
      "calibration": 94.7657142857143,
      "cc": 71.26430974729968,
      "cc_by_type": {
       "choice": 71.26430974729968
      },
      "cc_tiered": 73.07541667065746,
      "cc_tiered_open": 74.24692966214873,
      "cc_tiered_sealed": 71.90390367916619,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 71.26430974729968
     },
     "noul": {
      "accuracy": 0.8,
      "calibration": 89.81818181818183,
      "cc": 60.00000000000001,
      "cc_by_type": {
       "noul": 60.00000000000001
      },
      "cc_tiered": 49.02513093991858,
      "cc_tiered_open": 42.46376811594203,
      "cc_tiered_sealed": 55.58649376389514,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 60.00000000000001
     },
     "score": {
      "accuracy": 0.784,
      "calibration": 92.84768111111117,
      "cc": 67.94144144144141,
      "cc_by_type": {
       "score": 67.94144144144141
      },
      "cc_tiered": 61.05226552434946,
      "cc_tiered_open": 56.59117160303602,
      "cc_tiered_sealed": 65.5133594456629,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 67.94144144144141
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.005333333333333301,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": 61.050937711641836
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": null,
   "axes": {
    "calibration": 51.86891657577944,
    "cost": 78.6876666032308,
    "intelligence": 39.60641807869822,
    "speed": 91.86196998743578
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.28942929170579373,
      "ece_score": 42.114141658841255,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 322,
      "n_tvd": 0,
      "score": 42.114141658841255,
      "score_before_time_limit": 42.114141658841255,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.07466666666666666,
      "brier": 0.17653630763852338,
      "ece": 0.16426569406488034,
      "n": 375,
      "score": 67.14686118702393,
      "score_before_time_limit": 67.14686118702393,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.592,
      "interval90_width": 0.1486666666666667,
      "n": 375,
      "nrps": 0.19154766250281158,
      "score": 46.34574688147313,
      "score_before_time_limit": 46.34574688147313,
      "time_limit_failed_share": 0,
      "top_ece": 0.4407686999338628
     }
    },
    "score": 51.86891657577944
   },
   "capability": 45.737667327238825,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 771,182 input + 1,479 output tokens over 1,479 decisions = 522.421230561 tokens/decision. GPU-time cost USD 0.00513347100326989/1,000; old list-price cost USD 0.0157926369168357/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00513347100326989/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.04855274
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005133471003269886
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "decisor-4b",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-09",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 45.873028357917406,
    "I_sealed": 33.33980779947903,
    "base": 39.60641807869822,
    "excess": 5.445122814188888,
    "gap": 12.533220558438373,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 59.34553687284395,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 50.92796476879522,
       "judge": 56.49307844429796,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 60.469979296066256,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 17.803568904842024,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 31.0830694938378,
       "hard": 26.135294948151532,
       "judge": -4.274889595468578,
       "standard": 27.618054274190996
      }
     },
     "sealed|choice": {
      "cc": 53.06133527831807,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 66.66323695853484,
       "hard": 50.87225661226786,
       "judge": 49.8861047835991,
       "standard": 55.40138751238851
      }
     },
     "sealed|noul": {
      "cc": 46.958088120119044,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 66.66666666666667,
       "hard": 32.75862068965518,
       "judge": 43.28358208955223,
       "standard": 71.01449275362319
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 38.67363382263432,
       "hard": -1.500583889173468,
       "judge": -23.581442289262956,
       "standard": 15.816838400682286
      }
     }
    }
   },
   "jevbench_score": 36.84364855230138,
   "key": "decisor_4b",
   "last_measured_on": "2026-10-09",
   "listing": "ranked",
   "listing_note": "Cost re-estimated 12 Oct 2026 with the exact base-model reference (Qwen/Qwen3.5-4B, frozen DeepInfra 25 Sep snapshot, USD 0.03/0.15 per 1M, listing since deprecated) instead of the 9B proxy; composite A 33.52 → 35.34; Intelligence, Calibration, Speed unchanged.",
   "measured_in": "v1.6.3",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "1e7f195c2c41bbedec7d36b034001ef46476914f",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 51.86891657577944,
     "cost": 64.04636047691767,
     "intelligence": 39.60641807869822,
     "speed": 91.94556701350172
    },
    "board_release": "v1.6.43",
    "capability": 45.737667327238825,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 99,
    "capability_rank": 127,
    "composite_ci95": {
     "A": [
      24.74558117984167,
      45.875966423832466
     ],
     "B": [
      22.27609201764905,
      42.88135890458357
     ],
     "C": [
      17.184431374890043,
      31.858310016550323
     ]
    },
    "jevbench_score": 35.341565591640844,
    "measured_in": "v1.6.3",
    "method": "v1.6",
    "open_board_rank": 62,
    "rank": 92,
    "ranks": {
     "A": 92,
     "B": 91,
     "C": 93,
     "capability": 187
    },
    "scores": {
     "A": 35.341565591640844,
     "B": 32.590313637312704,
     "C": 24.542753883083925
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24632775298086926,
     "p50_s_raw": 0.04816387649043463,
     "p95_s_adjusted": 0.25937609854736365,
     "p95_s_raw": 0.054688049273681816
    },
    "views": {
     "Balanced 33:33:33": 36.3832685328225,
     "Emphasis on Accuracy 60:20:20": 30.68750459394931,
     "Emphasis on Cost 20:20:60": 37.81495456321552,
     "Emphasis on Speed 20:60:20": 42.6905897612171,
     "Intelligence only": 24.851733846686635
    }
   },
   "rank": 155,
   "ranked": true,
   "ranks": {
    "A": 155,
    "B": 153,
    "C": 156,
    "capability": 225
   },
   "repo": "https://huggingface.co/underlabs/decisor-4b",
   "rescored_in": "v1.7.0",
   "score_correction": {
    "decided_on": "2026-10-12",
    "published": {
     "axes_cost": 51.35109809189328,
     "composite_ci95": {
      "A": [
       23.54664202937882,
       43.43075242179646
      ],
      "B": [
       21.491047145911537,
       41.13808580347834
      ],
      "C": [
       16.351834742624177,
       30.160244737358653
      ]
     },
     "scores": {
      "A": 33.51963655001747,
      "B": 31.333881331679464,
      "C": 23.27752538195658
     },
     "usd_per_1000": 0.04184369844489519
    },
    "reason": "Standing exact-base rule (same reference as imajev-4b, aplomb-1, Uniphore): the model card names base_model Qwen/Qwen3.5-4B, so the frozen 25 Sep DeepInfra snapshot price applies instead of the Qwen3.5-9B proxy used at publication. Official scorer path re-run on the stored raw usage; only the cost axis moves.",
    "receipt": {
     "path": "data/jevbench-score-corrections-receipts/decisor_4b-new-boot.json",
     "sha256": "382b2a72e0cdab165a8b771051c58a368ec483283007a389420df783519045e1"
    }
   },
   "scores": {
    "A": 36.84364855230138,
    "B": 33.60090361786135,
    "C": 25.585867050209295
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24737461095210164,
    "p50_s_raw": 0.048687305476050824,
    "p95_s_adjusted": 0.2632981920789461,
    "p95_s_raw": 0.056649096039473074
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.68,
       "cc": 72.43717009494861,
       "cc_by_type": {
        "choice": 28.88888888888889,
        "noul": 100,
        "score": 88.42262139595695
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 72.43717009494861
      },
      "calibration_gen": {
       "accuracy": 0.4375,
       "cc": -2.696359073329281,
       "cc_by_type": {
        "choice": 30.041152263374478,
        "noul": 75,
        "score": -113.13022948336231
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.58,
       "cc": 11.41744858021391,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 43.99999999999999,
        "score": -77.22732905610623
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 11.41744858021391
      },
      "fairness": {
       "accuracy": 0.8571428571428571,
       "cc": 84.88687477950093,
       "cc_by_type": {
        "choice": 79.66101694915254,
        "noul": 100,
        "score": 74.99960738935026
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 84.88687477950093
      },
      "jaggedness": {
       "accuracy": 0.46296296296296297,
       "cc": 0.611805644933618,
       "cc_by_type": {
        "choice": 45.0358999752414,
        "noul": -21.42857142857143,
        "score": -21.77191161186911
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 0.611805644933618
      },
      "llm_router": {
       "accuracy": 0.7564935064935064,
       "cc": 66.40744255292894,
       "cc_by_type": {
        "choice": 63.078569517524315,
        "noul": 77.08333333333333,
        "score": 59.06042480792917
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 66.40744255292894
      },
      "long_state": {
       "accuracy": 0.38095238095238093,
       "cc": 6.470357542728832,
       "cc_by_type": {
        "choice": 22.413793103448274,
        "noul": 0,
        "score": -3.0027204752617775
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 6.470357542728832
      },
      "ml_usecase": {
       "accuracy": 0.4411764705882353,
       "cc": 14.984066446494696,
       "cc_by_type": {
        "choice": 15.78947368421053,
        "noul": 50,
        "score": -20.837274344726442
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 14.984066446494696
      },
      "multilingual": {
       "accuracy": 0.6385964912280702,
       "cc": 42.00929251734494,
       "cc_by_type": {
        "choice": 56.102117061021175,
        "noul": 39.02439024390243,
        "score": 30.901370247111203
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 42.00929251734494
      },
      "negation_pair_gen": {
       "accuracy": 0.5714285714285714,
       "cc": 14.507954819938357,
       "cc_by_type": {
        "choice": 59.99999999999999,
        "noul": 41.176470588235304,
        "score": -57.652606128420224
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 14.507954819938357
      },
      "option_order": {
       "accuracy": 0.7142857142857143,
       "cc": 63.599595551061675,
       "cc_by_type": {
        "choice": 63.599595551061675
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 63.599595551061675
      },
      "perturb_pair": {
       "accuracy": 0.7741935483870968,
       "cc": 76.27284247261942,
       "cc_by_type": {
        "choice": 62.26415094339623,
        "noul": 83.33333333333333,
        "score": 83.22104314112867
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 76.27284247261942
      },
      "usecase": {
       "accuracy": 0.5637254901960784,
       "cc": 38.82789670703383,
       "cc_by_type": {
        "choice": 49.21583271097835,
        "noul": 48,
        "score": 19.267857410123145
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 38.82789670703383
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5294117647058824,
       "cc": 11.482547816308236,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 66.66666666666667,
        "score": -75.07616607488481
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 11.482547816308236
      },
      "cs": {
       "accuracy": 0.7777777777777778,
       "cc": 56.32183908045977,
       "cc_by_type": {
        "choice": 79.3103448275862,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 56.32183908045977
      },
      "da": {
       "accuracy": 0.5625,
       "cc": 24.36962269643198,
       "cc_by_type": {
        "choice": 53.327571305099404,
        "noul": 50,
        "score": -30.218703215803465
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 24.36962269643198
      },
      "de": {
       "accuracy": 0.725,
       "cc": 57.51057081567085,
       "cc_by_type": {
        "choice": 77.76862906545902,
        "noul": 63.63636363636365,
        "score": 31.12671974518988
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 57.51057081567085
      },
      "el": {
       "accuracy": 0.4117647058823529,
       "cc": 13.019548214223159,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": -19.999999999999996,
        "score": 54.29673988076471
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 13.019548214223159
      },
      "en": {
       "accuracy": 0.613215859030837,
       "cc": 36.485554087248126,
       "cc_by_type": {
        "choice": 54.65150913423348,
        "noul": 53.73134328358209,
        "score": 1.073809843928808
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 36.485554087248126
      },
      "es": {
       "accuracy": 0.8,
       "cc": 69.53863836729614,
       "cc_by_type": {
        "choice": 87.95986622073579,
        "noul": 100,
        "score": 20.656048881152643
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 69.53863836729614
      },
      "fi": {
       "accuracy": 0.14285714285714285,
       "cc": 33.742518776617835,
       "cc_by_type": {
        "choice": -28.205128205128208,
        "noul": 100,
        "score": 29.43268453498171
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 33.742518776617835
      },
      "fr": {
       "accuracy": 0.75,
       "cc": 44.54072361680347,
       "cc_by_type": {
        "choice": 85.5421686746988,
        "noul": -33.333333333333336,
        "score": 81.41333550904494
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 44.54072361680347
      },
      "hi": {
       "accuracy": 0.5,
       "cc": 11.612135477060171,
       "cc_by_type": {
        "choice": 22.36598890942699,
        "noul": 66.66666666666667,
        "score": -54.196249144913146
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 11.612135477060171
      },
      "id": {
       "accuracy": 0.5,
       "cc": 34.42622950819673,
       "cc_by_type": {
        "choice": 34.42622950819673
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 34.42622950819673
      },
      "it": {
       "accuracy": 0.9230769230769231,
       "cc": 94.57878813420577,
       "cc_by_type": {
        "choice": 83.98576512455516,
        "noul": 100,
        "score": 99.75059927806217
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 94.57878813420577
      },
      "ja": {
       "accuracy": 0.5,
       "cc": -7.023794624927322,
       "cc_by_type": {
        "choice": 43.172849250197324,
        "noul": 0,
        "score": -64.24423312497929
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.5555555555555556,
       "cc": 0.9298891761473259,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -100,
        "score": 29.456334195108635
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 0.9298891761473259
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.96890180072307,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.90670540216922
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 99.96890180072307
      },
      "nl": {
       "accuracy": 0.7058823529411765,
       "cc": 50.07512343166962,
       "cc_by_type": {
        "choice": 56.86900958466453,
        "noul": 50,
        "score": 43.35636071034432
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 50.07512343166962
      },
      "no": {
       "accuracy": 0.75,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 43.62621783952097,
       "cc_by_type": {
        "choice": 69.84924623115577,
        "noul": 50,
        "score": 11.029407287407144
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 43.62621783952097
      },
      "pt": {
       "accuracy": 0.7083333333333334,
       "cc": 53.41628945933892,
       "cc_by_type": {
        "choice": 36.530324400564176,
        "noul": 75,
        "score": 48.718543977452576
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 53.41628945933892
      },
      "sv": {
       "accuracy": 0.25,
       "cc": 11.726974968514975,
       "cc_by_type": {
        "choice": 6.7357512953367875,
        "noul": -33.333333333333336,
        "score": 61.77850694354148
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 11.726974968514975
      },
      "tr": {
       "accuracy": 0.7272727272727273,
       "cc": 41.74690806177426,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 68.71898505488801
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 41.74690806177426
      },
      "uk": {
       "accuracy": 0.4444444444444444,
       "cc": -5.35189944151954,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -33.333333333333336,
        "score": -50.464300475096245
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 20.382580114218815,
       "cc_by_type": {
        "choice": 36.170212765957444,
        "noul": 33.33333333333333,
        "score": -8.355805756634327
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 20.382580114218815
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.652,
       "cc": 54.94735226989393,
       "cc_by_type": {
        "choice": 54.94735226989393
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 54.94735226989393
      },
      "noul": {
       "accuracy": 0.76,
       "cc": 52,
       "cc_by_type": {
        "noul": 52
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 52
      },
      "score": {
       "accuracy": 0.416,
       "cc": 6.985220437155759,
       "cc_by_type": {
        "score": 6.985220437155759
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 6.985220437155759
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5333333333333333,
       "cc": 13.54537966372772,
       "cc_by_type": {
        "choice": 36.39575971731449,
        "noul": 50,
        "score": -45.75962072613133
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 13.54537966372772
      },
      "code_linting": {
       "accuracy": 0.36363636363636365,
       "cc": 4.365900787853533,
       "cc_by_type": {
        "choice": -11.801242236024848,
        "noul": 19.999999999999996,
        "score": 4.8989445995854535
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 4.365900787853533
      },
      "customer_support": {
       "accuracy": 0.8461538461538461,
       "cc": 81.50290998545155,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 66.66666666666667,
        "score": 99.58119372447061
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 81.50290998545155
      },
      "demand_forecasting": {
       "accuracy": 0.3333333333333333,
       "cc": -27.435863585569564,
       "cc_by_type": {
        "choice": 35.064935064935064,
        "noul": -100,
        "score": -17.37252582164377
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5714285714285714,
       "cc": 31.878796844984276,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 19.999999999999996,
        "score": 8.423275780854468
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 31.878796844984276
      },
      "feature_extraction": {
       "accuracy": 0.625,
       "cc": 35.95984643622633,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 33.33333333333333,
        "score": 37.04620597534565
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 35.95984643622633
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 35.0411180236903,
       "cc_by_type": {
        "choice": 46.80851063829788,
        "noul": 60.00000000000001,
        "score": -1.6851565672269864
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 35.0411180236903
      },
      "gaming": {
       "accuracy": 0.35294117647058826,
       "cc": 4.331399313402698,
       "cc_by_type": {
        "choice": 29.742388758782198,
        "noul": 0,
        "score": -16.748190818574106
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 4.331399313402698
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 51.88432725992252,
       "cc_by_type": {
        "choice": 21.052631578947366,
        "noul": 100,
        "score": 34.600350200820216
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 51.88432725992252
      },
      "knowledge_graphs": {
       "accuracy": 0.42857142857142855,
       "cc": 11.978713611397161,
       "cc_by_type": {
        "choice": 33.62831858407079,
        "noul": -14.28571428571429,
        "score": 16.593536535834986
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 11.978713611397161
      },
      "lead_generation": {
       "accuracy": 0.7407407407407407,
       "cc": 55.487042701031726,
       "cc_by_type": {
        "choice": 69.6969696969697,
        "noul": 100,
        "score": -3.235841593874511
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 55.487042701031726
      },
      "legal_compliance": {
       "accuracy": 0.4666666666666667,
       "cc": 18.106251584114915,
       "cc_by_type": {
        "choice": 47.368421052631575,
        "noul": -25,
        "score": 31.950333699713173
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 18.106251584114915
      },
      "llm_guardrails": {
       "accuracy": 0.7083333333333334,
       "cc": 51.42465114663353,
       "cc_by_type": {
        "choice": 50.81967213114754,
        "noul": 42.85714285714286,
        "score": 60.59713845161019
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 51.42465114663353
      },
      "model_routing": {
       "accuracy": 0.8029556650246306,
       "cc": 70.31448359900337,
       "cc_by_type": {
        "choice": 77.35549228630074,
        "noul": 86.44067796610169,
        "score": 47.147280544607696
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 70.31448359900337
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 31.245043349248153,
       "cc_by_type": {
        "choice": 31.81818181818182,
        "noul": 0,
        "score": 61.91694822956265
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 31.245043349248153
      },
      "recruiting": {
       "accuracy": 0.5882352941176471,
       "cc": 52.695732503224406,
       "cc_by_type": {
        "choice": 62.382445141065844,
        "noul": 50,
        "score": 45.70475236860735
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 52.695732503224406
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 22.497365563076894,
       "cc_by_type": {
        "choice": 41.32029339853301,
        "noul": 33.33333333333333,
        "score": -7.1615300426356665
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 22.497365563076894
      },
      "scientific_discovery": {
       "accuracy": 0.5238095238095238,
       "cc": 23.782218083000426,
       "cc_by_type": {
        "choice": 41.747572815533985,
        "noul": 0,
        "score": 29.59908143346729
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 23.782218083000426
      },
      "search_retrieval": {
       "accuracy": 0.75,
       "cc": 74.1814325078106,
       "cc_by_type": {
        "choice": 88.39458413926498,
        "noul": 100,
        "score": 34.1497133841668
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 74.1814325078106
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      46.9497762071451,
      56.41045180978028
     ],
     "I": [
      34.24026015889371,
      44.09619091401736
     ],
     "capability": [
      41.14796787605721,
      49.56906133902689
     ],
     "composite": null,
     "seed": 21
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 51.86891657577944,
     "capability": 45.737667327238825,
     "intelligence": 39.60641807869822
    },
    "per_type": {
     "choice": {
      "accuracy": 0.652,
      "calibration": 42.114141658841255,
      "cc": 54.94735226989393,
      "cc_by_type": {
       "choice": 54.94735226989393
      },
      "cc_tiered": 56.20343607558101,
      "cc_tiered_open": 59.34553687284395,
      "cc_tiered_sealed": 53.06133527831807,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.94735226989393
     },
     "noul": {
      "accuracy": 0.76,
      "calibration": 67.14686118702393,
      "cc": 52,
      "cc_by_type": {
       "noul": 52
      },
      "cc_tiered": 53.71403370809265,
      "cc_tiered_open": 60.469979296066256,
      "cc_tiered_sealed": 46.958088120119044,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 52
     },
     "score": {
      "accuracy": 0.416,
      "calibration": 46.34574688147313,
      "cc": 6.985220437155759,
      "cc_by_type": {
       "score": 6.985220437155759
      },
      "cc_tiered": 8.901784452421012,
      "cc_tiered_open": 17.803568904842024,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 6.985220437155759
     }
    },
    "raw_on_A": {
     "calibration": 54.13393694270948,
     "capability": 49.11000935691333,
     "intelligence": 44.08608177111718
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 38.54000835465314,
    "Emphasis on Accuracy 60:20:20": 31.58191220801446,
    "Emphasis on Cost 20:20:60": 42.248147271637215,
    "Emphasis on Speed 20:60:20": 44.429010411180684,
    "Intelligence only": 24.851733846686635
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "krishna765",
   "axes": {
    "calibration": 88.59116686443139,
    "cost": 78.00338436233434,
    "intelligence": 38.62465958342338,
    "speed": 89.8569997500675
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06995357456541221,
      "ece_score": 86.00928508691756,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 86.00928508691756,
      "score_before_time_limit": 86.00928508691756,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3207547169811321,
      "brier": 0.11609579904406445,
      "ece": 0.05066343452944609,
      "n": 371,
      "score": 89.86731309411078,
      "score_before_time_limit": 89.86731309411078,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9539295392953929,
      "interval90_width": 0.47515808491418265,
      "n": 369,
      "nrps": 0.10083978781514244,
      "score": 89.89690241226586,
      "score_before_time_limit": 89.89690241226586,
      "time_limit_failed_share": 0,
      "top_ece": 0.050611081969770266
     }
    },
    "score": 88.59116686443139
   },
   "capability": 63.60791322392738,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 813,225 input + 0 output tokens over 1,477 decisions = 550.592417062 tokens/decision. GPU-time cost USD 0.00541028970926366/1,000; old list-price cost USD 0.0165177725118483/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00541028970926366/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.0162645
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005410289709263655
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "deck-4B v1.0 (krishna765, JevK5 + LoRA, FP8 weight-only)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 38.79947935146771,
    "I_sealed": 38.44983981537905,
    "base": 38.62465958342338,
    "excess": -2.2237481063495843,
    "gap": 0.3496395360886595,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 58.68740862543909,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 60.36489462094999,
       "judge": 64.40342781806196,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 25.017253278122844,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": 0,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 32.6937761508412,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.877671536622636,
       "hard": 34.178023262130495,
       "judge": 35.59214965227232,
       "standard": 15.785773983225237
      }
     },
     "sealed|choice": {
      "cc": 59.157265682164855,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 69.40359175227256,
       "hard": 56.22171945701356,
       "judge": 59.031657355679705,
       "standard": 60.09360758714129
      }
     },
     "sealed|noul": {
      "cc": 21.704287246216186,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48,
       "hard": 0.9708737864077666,
       "judge": 31.578947368421062,
       "standard": 35.2112676056338
      }
     },
     "sealed|score": {
      "cc": 34.48796651775611,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.00896406653321,
       "hard": 26.22895515401159,
       "judge": 35.59993017640196,
       "standard": 33.57754498288781
      }
     }
    }
   },
   "jevbench_score": 39.04972547531234,
   "key": "deck-4b-v1-0",
   "last_measured_on": "2026-10-05",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9325396825396826,
    "decisive_rate": 0.6792452830188679,
    "supported": true,
    "valid": 371
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.59116686443139,
     "cost": 63.46145558281372,
     "intelligence": 38.62465958342338,
     "speed": 89.93848373337222
    },
    "board_release": "v1.6.43",
    "capability": 63.60791322392738,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 43,
    "capability_rank": 67,
    "composite_ci95": {
     "A": [
      23.964977734953006,
      50.38055037709117
     ],
     "B": [
      20.750561783148846,
      45.824903274791566
     ],
     "C": [
      16.642345649272922,
      34.98649331742443
     ]
    },
    "jevbench_score": 37.26498528662829,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 58,
    "rank": 88,
    "ranks": {
     "A": 88,
     "B": 89,
     "C": 89,
     "capability": 109
    },
    "scores": {
     "A": 37.26498528662829,
     "B": 33.17299510735835,
     "C": 25.87846200460298
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.30241732355207207,
     "p50_s_raw": 0.07620866177603602,
     "p95_s_adjusted": 0.3353860241360962,
     "p95_s_raw": 0.09269301206804811
    },
    "views": {
     "Balanced 33:33:33": 33.95960169128873,
     "Emphasis on Accuracy 60:20:20": 28.553229571667234,
     "Emphasis on Cost 20:20:60": 35.46476300764428,
     "Emphasis on Speed 20:60:20": 39.807414011248,
     "Intelligence only": 23.04910072446208
    }
   },
   "provenance": {
    "carried_metadata_from": null,
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 152,
   "ranked": true,
   "ranks": {
    "A": 152,
    "B": 152,
    "C": 153,
    "capability": 140
   },
   "repo": "https://github.com/krishna-gogineni-765/deck",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 39.04972547531234,
    "B": 34.289051680976144,
    "C": 27.11786491341135
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.30445735771209004,
    "p50_s_raw": 0.07722867885604501,
    "p95_s_adjusted": 0.33944823201745744,
    "p95_s_raw": 0.09472411600872874
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.64482190603961,
      89.1506890221124
     ],
     "I": [
      32.29690931230169,
      43.8177901770495
     ],
     "capability": [
      58.30292793082468,
      65.68855559803174
     ],
     "composite": null,
     "seed": 33
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.59116686443139,
     "capability": 63.60791322392738,
     "intelligence": 38.62465958342338
    },
    "per_type": {
     "choice": {
      "accuracy": 0.688,
      "calibration": 86.00928508691756,
      "cc": 60.099656129515054,
      "cc_by_type": {
       "choice": 60.099656129515054
      },
      "cc_tiered": 58.922337153801976,
      "cc_tiered_open": 58.68740862543909,
      "cc_tiered_sealed": 59.157265682164855,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.099656129515054
     },
     "noul": {
      "accuracy": 0.6266666666666667,
      "calibration": 89.86731309411078,
      "cc": 25.333333333333343,
      "cc_by_type": {
       "noul": 25.333333333333343
      },
      "cc_tiered": 23.360770262169517,
      "cc_tiered_open": 25.017253278122844,
      "cc_tiered_sealed": 21.704287246216186,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 25.333333333333343
     },
     "score": {
      "accuracy": 0.5546666666666666,
      "calibration": 89.89690241226586,
      "cc": 35.52454801522435,
      "cc_by_type": {
       "score": 35.52454801522435
      },
      "cc_tiered": 33.590871334298654,
      "cc_tiered_open": 32.6937761508412,
      "cc_tiered_sealed": 34.48796651775611,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.52454801522435
     }
    },
    "raw_on_A": {
     "calibration": 89.0005990160775,
     "capability": 66.05004010202693,
     "intelligence": 43.09948118797635
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 35.920446539712586,
    "Emphasis on Accuracy 60:20:20": 29.361819579181436,
    "Emphasis on Cost 20:20:60": 39.53065390437495,
    "Emphasis on Speed 20:60:20": 41.38518912113375,
    "Intelligence only": 23.04910072446208
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "krishna765",
   "axes": {
    "calibration": 82.15636027953371,
    "cost": 55.01396479280669,
    "intelligence": 73.02512554551848,
    "speed": 86.37611187702757
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10701823999728471,
      "ece_score": 78.59635200054306,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 78.59635200054306,
      "score_before_time_limit": 78.59635200054306,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.037735849056603765,
      "brier": 0.059638464379526884,
      "ece": 0.052855966211929055,
      "n": 371,
      "score": 89.42880675761418,
      "score_before_time_limit": 89.42880675761418,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8482384823848238,
      "interval90_width": 0.08897922312556458,
      "n": 369,
      "nrps": 0.06373815398690165,
      "score": 78.44392208044394,
      "score_before_time_limit": 78.44392208044394,
      "time_limit_failed_share": 0,
      "top_ece": 0.18369170220210976
     }
    },
    "score": 82.15636027953371
   },
   "capability": 77.5907429125261,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 779,964 input + 0 output tokens over 1,477 decisions = 528.073121192 tokens/decision. GPU-time cost USD 0.0315889002833892/1,000; old list-price cost USD 0.0475265809072444/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0315889002833892/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.04679784
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03158890028338916
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "deck-31B (krishna765, frozen Gemma-4-31B-it, TorchAO FP8 dynamic)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 70.23424531362603,
    "I_sealed": 75.81600577741091,
    "base": 73.02512554551848,
    "excess": -8.155148106223118,
    "gap": -5.581760463784875,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 72.33184897895812,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 69.80182447310474,
       "judge": 64.40342781806196,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 77.39268461007592,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 60.00000000000001,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 60.978202351844075,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.27912483548366,
       "hard": 65.74514134257689,
       "judge": 59.223722663075364,
       "standard": 47.92558266171173
      }
     },
     "sealed|choice": {
      "cc": 83.85398241235396,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 90.68804966373514,
       "hard": 76.5837104072398,
       "judge": 87.89571694599628,
       "standard": 88.91489099642814
      }
     },
     "sealed|noul": {
      "cc": 78.05289930692999,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 96,
       "hard": 66.99029126213591,
       "judge": 81.57894736842107,
       "standard": 85.91549295774648
      }
     },
     "sealed|score": {
      "cc": 65.54113561294878,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 75.42107738087628,
       "hard": 61.37893873388316,
       "judge": 71.30554200754482,
       "standard": 60.278948895222165
      }
     }
    }
   },
   "jevbench_score": 71.91617991839077,
   "key": "deck31b",
   "last_measured_on": "2026-10-05",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9439775910364145,
    "decisive_rate": 0.9622641509433962,
    "supported": true,
    "valid": 371
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.15636027953371,
     "cost": 49.69190284016272,
     "intelligence": 73.02512554551848,
     "speed": 86.69970536196755
    },
    "board_release": "v1.6.43",
    "capability": 77.5907429125261,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 10,
    "capability_rank": 19,
    "composite_ci95": {
     "A": [
      67.07103330487536,
      69.73758073723451
     ],
     "B": [
      67.10121070795465,
      70.84803238839223
     ],
     "C": [
      67.07103330487536,
      69.73758073723451
     ]
    },
    "jevbench_score": 68.68200118286292,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 14,
    "rank": 31,
    "ranks": {
     "A": 31,
     "B": 23,
     "C": 27,
     "capability": 35
    },
    "scores": {
     "A": 68.68200118286292,
     "B": 69.34459478009123,
     "C": 68.68200118286292
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4049364184960723,
     "p50_s_raw": 0.12746820924803615,
     "p95_s_adjusted": 0.5280105819925666,
     "p95_s_raw": 0.18900529099628327
    },
    "views": {
     "Balanced 33:33:33": 65.98518077730212,
     "Emphasis on Accuracy 60:20:20": 68.52137025063348,
     "Emphasis on Cost 20:20:60": 58.32800868887079,
     "Emphasis on Speed 20:60:20": 72.85260809439177,
     "Intelligence only": 72.71357044848186
    }
   },
   "provenance": {
    "carried_metadata_from": null,
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 59,
   "ranked": true,
   "ranks": {
    "A": 59,
    "B": 36,
    "C": 46,
    "capability": 47
   },
   "repo": "https://github.com/krishna-gogineni-765/deck31b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.91617991839077,
    "B": 72.13526639755992,
    "C": 71.91617991839077
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4106213027611375,
    "p50_s_raw": 0.13031065138056874,
    "p95_s_adjusted": 0.5609799453057348,
    "p95_s_raw": 0.20548997265286742
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.16081966059086,
      84.92278583205757
     ],
     "I": [
      68.03652261338961,
      76.64977164564989
     ],
     "capability": [
      73.77453269228936,
      80.24074177369353
     ],
     "composite": null,
     "seed": 34
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.15636027953371,
     "capability": 77.5907429125261,
     "intelligence": 73.02512554551848
    },
    "per_type": {
     "choice": {
      "accuracy": 0.86,
      "calibration": 78.59635200054306,
      "cc": 82.09599954529521,
      "cc_by_type": {
       "choice": 82.09599954529521
      },
      "cc_tiered": 78.09291569565605,
      "cc_tiered_open": 72.33184897895812,
      "cc_tiered_sealed": 83.85398241235396,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 82.09599954529521
     },
     "noul": {
      "accuracy": 0.8986666666666666,
      "calibration": 89.42880675761418,
      "cc": 79.73333333333332,
      "cc_by_type": {
       "noul": 79.73333333333332
      },
      "cc_tiered": 77.72279195850295,
      "cc_tiered_open": 77.39268461007592,
      "cc_tiered_sealed": 78.05289930692999,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 79.73333333333332
     },
     "score": {
      "accuracy": 0.72,
      "calibration": 78.44392208044394,
      "cc": 64.7077492239505,
      "cc_by_type": {
       "score": 64.7077492239505
      },
      "cc_tiered": 63.25966898239643,
      "cc_tiered_open": 60.978202351844075,
      "cc_tiered_sealed": 65.54113561294878,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 64.7077492239505
     }
    },
    "raw_on_A": {
     "calibration": 79.57362236322788,
     "capability": 77.19870727297797,
     "intelligence": 74.82379218272806
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.04743007350025,
    "Emphasis on Accuracy 60:20:20": 70.58535021170198,
    "Emphasis on Cost 20:20:60": 62.65444276205019,
    "Emphasis on Speed 20:60:20": 75.07175573127698,
    "Intelligence only": 73.02512554551848
   }
  },
  {
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "LibertAI",
   "axes": {
    "calibration": 29.6278299611924,
    "cost": 86.84458976989441,
    "intelligence": 6.531204174677637,
    "speed": 84.83282651822053
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.5370261325853646,
      "ece_score": 0,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 0,
      "score_before_time_limit": 0,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2264150943396226,
      "brier": 0.34735578114832993,
      "ece": 0.30478924226600856,
      "n": 371,
      "score": 39.04215154679829,
      "score_before_time_limit": 39.04215154679829,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.6233062330623306,
      "interval90_width": 0.4112466124661248,
      "n": 369,
      "nrps": 0.24237776576192704,
      "score": 49.84133833677892,
      "score_before_time_limit": 49.84133833677892,
      "time_limit_failed_share": 0,
      "top_ece": 0.3803977337512473
     }
    },
    "score": 29.6278299611924
   },
   "capability": 18.07951706793502,
   "class": "system-one-open",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.8B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 737,323 input + 0 output tokens over 1,624 decisions = 454.016625616 tokens/decision. GPU-time cost USD 0.00274481876637149/1,000; old list-price cost USD 0.00454016625615764/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00274481876637149/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0027448187663714914
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Deem 0.8B v1",
   "endpoint_condition": "evaluator-owned H100 GPU",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 8.352332961291632,
    "I_sealed": 4.710075388063642,
    "base": 6.531204174677637,
    "excess": 1.0688699307897456,
    "gap": 3.6422575732279894,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 25.056998883874897,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 41.621621621621614,
       "hard": 9.405473419314262,
       "judge": 40.672379696769944,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0,
       "hard": -39.13043478260869,
       "judge": -9.999999999999998,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 23.118548982848488,
       "hard": -12.673038757528188,
       "judge": 7.843499385378882,
       "standard": -67.26014388289686
      }
     },
     "sealed|choice": {
      "cc": 14.130226164190928,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 32.15579040721308,
       "hard": 3.7895927601809762,
       "judge": 17.132216014897597,
       "standard": 21.29572607463975
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 28.000000000000004,
       "hard": -39.80582524271845,
       "judge": -2.631578947368418,
       "standard": 4.225352112676051
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -7.688289022450467,
       "hard": -26.74875998927384,
       "judge": -33.081610869032005,
       "standard": -11.930482263268317
      }
     }
    }
   },
   "jevbench_score": 0.3247463549936592,
   "key": "deem-0.8-v1",
   "last_measured_on": "2026-10-04",
   "licence": "Not independently recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5923344947735192,
    "decisive_rate": 0.7735849056603774,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 29.6278299611924,
     "cost": 80.28784730320886,
     "intelligence": 6.531204174677637,
     "speed": 85.44505463939521
    },
    "board_release": "v1.6.43",
    "capability": 18.07951706793502,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 135,
    "capability_rank": 165,
    "composite_ci95": {
     "A": [
      0.12746211676468558,
      0.7725020253140645
     ],
     "B": [
      0.08887678609852033,
      0.587439805908629
     ],
     "C": [
      0.08851535886436501,
      0.5364597398014336
     ]
    },
    "jevbench_score": 0.32342918922131697,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 165,
    "rank": 209,
    "ranks": {
     "A": 209,
     "B": 209,
     "C": 209,
     "capability": 237
    },
    "scores": {
     "A": 0.32342918922131697,
     "B": 0.23429153017801835,
     "C": 0.2246036036259146
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4647796718403697,
     "p50_s_raw": 0.15738983592018485,
     "p95_s_adjusted": 0.6141117558348923,
     "p95_s_raw": 0.23205587791744614
    },
    "views": {
     "Balanced 33:33:33": 0.28875743313428337,
     "Emphasis on Accuracy 60:20:20": 0.17645226737338762,
     "Emphasis on Cost 20:20:60": 0.4219666221718261,
     "Emphasis on Speed 20:60:20": 0.4251280933682276,
     "Intelligence only": 0.11143965867359962
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 250,
   "ranked": true,
   "ranks": {
    "A": 250,
    "B": 250,
    "C": 250,
    "capability": 278
   },
   "repo": "https://huggingface.co/LibertAIDAI/deem-0.8-v1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.3247463549936592,
    "B": 0.23484353404797886,
    "C": 0.22551830207893003
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4788308489136398,
    "p50_s_raw": 0.1644154244568199,
    "p95_s_adjusted": 0.6863335451111197,
    "p95_s_raw": 0.2681667725555598
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "LibertAIDAI/deem-0.8-v1 @ 8cbabbb2c4a7ef13c6b43f0ef3ae4157983c6d21; root model.safetensors SHA-256 80438125177c0681e855158d46720cbbd4ff07ea620cd55b0f456aafbfcf778d. Author Python GPU server, default DEEM_N_ORDERS=1, raw softmax (temperature 1.0). A four-field shim translates the wire format. The submitted interface accepts choice labels without descriptions and noul instructions without criteria; score receives full level descriptions. The author Rust/AVX-512 latency claim was not measured.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      25.404113930608833,
      32.82152927572935
     ],
     "I": [
      4.660400387407388,
      9.084320195556709
     ],
     "capability": [
      15.752622734868103,
      20.176991128075684
     ],
     "composite": null,
     "seed": 35
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 29.6278299611924,
     "capability": 18.07951706793502,
     "intelligence": 6.531204174677637
    },
    "per_type": {
     "choice": {
      "accuracy": 0.34933333333333333,
      "calibration": 0,
      "cc": 16.78902645813397,
      "cc_by_type": {
       "choice": 16.78902645813397
      },
      "cc_tiered": 19.593612524032913,
      "cc_tiered_open": 25.056998883874897,
      "cc_tiered_sealed": 14.130226164190928,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 16.78902645813397
     },
     "noul": {
      "accuracy": 0.4533333333333333,
      "calibration": 39.04215154679829,
      "cc": -9.333333333333337,
      "cc_by_type": {
       "noul": -9.333333333333337
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.30933333333333335,
      "calibration": 49.84133833677892,
      "cc": -19.150447666018742,
      "cc_by_type": {
       "score": -19.150447666018742
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 29.6549349776408,
     "capability": 17.995438687555687,
     "intelligence": 6.335942397470573
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.2901583874635193,
    "Emphasis on Accuracy 60:20:20": 0.17676518563104343,
    "Emphasis on Cost 20:20:60": 0.42775655929259215,
    "Emphasis on Speed 20:60:20": 0.4265884340784719,
    "Intelligence only": 0.11143965867359962
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 595,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "DeepSeek",
   "axes": {
    "calibration": 94.76,
    "cost": 19.56,
    "intelligence": 84.48,
    "speed": 65.99
   },
   "calibration": {
    "score": 94.76
   },
   "capability": 89.62,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     6.79,
     7.03
    ],
    "B": [
     7.41,
     7.83
    ],
    "C": [
     6.79,
     7.03
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.15/M input, 0.6/M output (spec deepseek-flash.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.480105076142132
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 34,
    "n_over_limit_answered_ok": 31,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "DeepSeek V4.1 Flash (thinking default)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 82.33,
    "I_sealed": 81.49,
    "base": 84.48,
    "per_type_split": {
     "open|choice": {
      "cc": 95.4703,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100,
       "hard": 88.6757,
       "judge": 100,
       "standard": 100
      }
     },
     "open|noul": {
      "cc": 93.5217,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.3043,
       "judge": 90,
       "standard": 100
      }
     },
     "open|score": {
      "cc": 57.9899,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 41.9091,
       "hard": 71.9722,
       "judge": 68.4237,
       "standard": 22.4148
      }
     },
     "sealed|choice": {
      "cc": 96.1218,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 100,
       "hard": 94.1829,
       "judge": 100,
       "standard": 92.2431
      }
     },
     "sealed|noul": {
      "cc": 87.8636,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 100,
       "hard": 90.9091,
       "judge": 80,
       "standard": 87.5
      }
     },
     "sealed|score": {
      "cc": 60.4774,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 38.6944,
       "hard": 40.4217,
       "judge": 94.5455,
       "standard": 60.378
      }
     }
    }
   },
   "jevbench_score": 6.9,
   "key": "deepseek-flash",
   "last_measured_on": "2026-10-06",
   "licence": "open weights, proprietary API route",
   "listing": "ranked",
   "method": "v1.7",
   "open": "weights",
   "previous": {
    "api_board_rank": 40,
    "axes": {
     "calibration": 100,
     "cost": 19.56,
     "intelligence": 94.71,
     "speed": 68.07
    },
    "board_release": "v1.6.43",
    "capability": 97.35,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      7.02,
      7.13
     ],
     "B": [
      7.78,
      7.99
     ],
     "C": [
      7.02,
      7.13
     ]
    },
    "jevbench_score": 7.09,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 180,
    "ranks": {
     "A": 180,
     "B": 174,
     "C": 172,
     "capability": 4
    },
    "scores": {
     "A": 7.09,
     "B": 7.89,
     "C": 7.09
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 595,
     "p50_s_adjusted": 2.038,
     "p50_s_raw": 2.038,
     "p95_s_adjusted": 7.648,
     "p95_s_raw": 7.648
    },
    "views": null
   },
   "rank": 225,
   "ranked": true,
   "ranks": {
    "A": 225,
    "B": 217,
    "C": 216,
    "capability": 4
   },
   "repo": "https://benchmarkheaven.com/jev-models/deepseek-flash",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 6.9,
    "B": 7.61,
    "C": 6.9
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 2.205,
    "p50_s_raw": 2.205,
    "p95_s_adjusted": 11.414,
    "p95_s_raw": 11.414
   },
   "status": {
    "answered_ok": 595,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "c9078842b58b17d1e14e86c940cbe4291066ec8a620c430aac3b4385e656feb0"
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Hangzhi Zhou (@Hangzhi)",
   "axes": {
    "calibration": 61.31489461738089,
    "cost": 71.2270380297646,
    "intelligence": 54.860121638521335,
    "speed": 86.28217783727928
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.2516031734669799,
      "ece_score": 49.67936530660402,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 49.67936530660402,
      "score_before_time_limit": 49.67936530660402,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.07027027027027022,
      "brier": 0.14489436428485655,
      "ece": 0.14040055215319622,
      "n": 370,
      "score": 71.91988956936075,
      "score_before_time_limit": 71.91988956936075,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.6811989100817438,
      "interval90_width": 0.0510899182561308,
      "n": 367,
      "nrps": 0.11223140401478648,
      "score": 62.3454289761779,
      "score_before_time_limit": 62.3454289761779,
      "time_limit_failed_share": 0,
      "top_ece": 0.3204300082308278
     }
    },
    "score": 61.31489461738089
   },
   "capability": 58.08750812795111,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 763,351 input + 7,370 output tokens over 1,477 decisions = 521.815165877 tokens/decision. GPU-time cost USD 0.00910120155482596/1,000; old list-price cost USD 0.0480112322274881/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00910120155482596/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.047275059999999994
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00910120155482596
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Diffusion Jev (DiffusionGemma 26B-A4B on patched SGLang, 48-step diffusion readout, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. github.com/Hangzhi/diffusion-jev-sglang@d61f0a9c0c9b6d78631aa250125dc4b20e9ea0a2 serving google/diffusiongemma-26B-A4B-it@f7f5b7f5fa82ffc52addd066915886d497f5517b on SGLang@ef90bf677e4e6b280bf796f1c5f4ef0f05dda27f (their open PR 34061) with their own version-checked readout and context-admission patch. Built from THEIR OWN tested container recipe (deployment/modal_app.py + deployment/gemma-requirements.txt), which their doc names as the recipe to use; their `engine` extra and top-level Dockerfile target the older LLaDA deployment and their doc says plainly not to use them as a DiffusionGemma installer, so they were not used. SGLang is a pure-python install imported through PYTHONPATH, no source compile. Served with their own scripts/launch_diffusiongemma.py: native Gemma4Renoise, a 256-position canvas, up to 48 adaptive denoising steps, seed 42, thinking disabled, 8,192-token context, Triton attention, no CUDA graphs, no prefix cache, T=1 softmax over the options offered - their configuration, unchanged, on one H100 80 GB (Lium) against the A100 80 GB they tested. THE AUTHOR'S OWN STATEMENT, CARRIED UNSOFTENED: these are self-conditioned denoiser scores, NOT independent masked-token likelihoods and NOT calibrated probabilities of correctness; this is not a one-forward-pass or zero-generation method. There are no new trained weights - it is an inference configuration over Google's unchanged checkpoint. A format mismatch returns their own HTTP 503 and is recorded invalid, the same way for everyone. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 55, GitHub #30",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 53.207343059355594,
    "I_sealed": 56.51290021768708,
    "base": 54.860121638521335,
    "excess": -5.878944800769732,
    "gap": -3.3055571583314887,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 66.57316141515619,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 60.36489462094999,
       "judge": 56.49307844429796,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 51.94202898550725,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 19.999999999999996,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 41.10683877740336,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 42.890763672952836,
       "hard": 43.8594983288807,
       "judge": 51.92844064579445,
       "standard": 18.477154424087306
      }
     },
     "sealed|choice": {
      "cc": 67.58861736481802,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 59.27601809954751,
       "judge": 70.2048417132216,
       "standard": 73.39573839142753
      }
     },
     "sealed|noul": {
      "cc": 53.955192987254144,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 32.038834951456316,
       "judge": 65.78947368421053,
       "standard": 69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 47.994890300989084,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.90991267549208,
       "hard": 40.176230341477435,
       "judge": 50.19410623577225,
       "standard": 49.875875130586145
      }
     }
    }
   },
   "jevbench_score": 66.48171557572483,
   "key": "diffusion-jev",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0 (model card metadata); linked Gemma 4 model terms",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "google/diffusiongemma-26B-A4B-it@f7f5b7f5fa82ffc52addd066915886d497f5517b",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8546511627906976,
    "decisive_rate": 0.9297297297297298,
    "supported": true,
    "valid": 370
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 61.31489461738089,
     "cost": 49.559714426383636,
     "intelligence": 54.860121638521335,
     "speed": 86.79193858661813
    },
    "board_release": "v1.6.43",
    "capability": 58.08750812795111,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 66,
    "capability_rank": 91,
    "composite_ci95": {
     "A": [
      52.773263175899025,
      61.260192333972526
     ],
     "B": [
      50.91942517195592,
      60.65419970930148
     ],
     "C": [
      36.648099427707656,
      60.66776170099464
     ]
    },
    "jevbench_score": 59.330428224994314,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 34,
    "rank": 58,
    "ranks": {
     "A": 58,
     "B": 56,
     "C": 52,
     "capability": 144
    },
    "scores": {
     "A": 59.330428224994314,
     "B": 58.15811643913557,
     "C": 49.60078097158421
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4113455285318196,
     "p50_s_raw": 0.1306727642659098,
     "p95_s_adjusted": 0.5088612267747521,
     "p95_s_raw": 0.17943061338737604
    },
    "views": {
     "Balanced 33:33:33": 59.033372905673865,
     "Emphasis on Accuracy 60:20:20": 56.86619781829752,
     "Emphasis on Cost 20:20:60": 54.41034813835743,
     "Emphasis on Speed 20:60:20": 67.31868662948479,
     "Intelligence only": 53.898210717803785
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 104,
   "ranked": true,
   "ranks": {
    "A": 104,
    "B": 99,
    "C": 93,
    "capability": 177
   },
   "repo": "https://huggingface.co/google/diffusiongemma-26B-A4B-it",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 66.48171557572483,
    "B": 63.779499738988385,
    "C": 55.57932264337709
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.41225921185687187,
    "p50_s_raw": 0.13112960592843592,
    "p95_s_adjusted": 0.5709681035950779,
    "p95_s_raw": 0.21048405179753896
   },
   "status": {
    "answered_ok": 1474,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/diffusiongemma-26B-A4B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      56.00277250017865,
      64.96308698031199
     ],
     "I": [
      48.25688155312269,
      59.81154987802872
     ],
     "capability": [
      52.99878343969597,
      61.730417849443256
     ],
     "composite": null,
     "seed": 50
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 61.31489461738089,
     "capability": 58.08750812795111,
     "intelligence": 54.860121638521335
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7533333333333333,
      "calibration": 49.67936530660402,
      "cc": 68.45485634171062,
      "cc_by_type": {
       "choice": 68.45485634171062
      },
      "cc_tiered": 67.0808893899871,
      "cc_tiered_open": 66.57316141515619,
      "cc_tiered_sealed": 67.58861736481802,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 68.45485634171062
     },
     "noul": {
      "accuracy": 0.784,
      "calibration": 71.91988956936075,
      "cc": 56.800000000000004,
      "cc_by_type": {
       "noul": 56.800000000000004
      },
      "cc_tiered": 52.9486109863807,
      "cc_tiered_open": 51.94202898550725,
      "cc_tiered_sealed": 53.955192987254144,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 56.800000000000004
     },
     "score": {
      "accuracy": 0.6106666666666667,
      "calibration": 62.3454289761779,
      "cc": 47.39565350609251,
      "cc_by_type": {
       "score": 47.39565350609251
      },
      "cc_tiered": 44.550864539196226,
      "cc_tiered_open": 41.10683877740336,
      "cc_tiered_sealed": 47.994890300989084,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 47.39565350609251
     }
    },
    "raw_on_A": {
     "calibration": 57.69524290701848,
     "capability": 57.542251645635915,
     "intelligence": 57.38926038425334
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.021333333333333315,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.4030894853535,
    "Emphasis on Accuracy 60:20:20": 62.25562961817577,
    "Emphasis on Cost 20:20:60": 69.5053651387668,
    "Emphasis on Speed 20:60:20": 74.58520244386014,
    "Intelligence only": 54.860121638521335
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "DOPEST-studio (Hugging Face)",
   "axes": {
    "calibration": 90.20395849508765,
    "cost": 52.643215806695544,
    "intelligence": 73.86361411900745,
    "speed": 81.97358858059908
   },
   "base_model": "google/gemma-4-31B-it",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04220545442376594,
      "ece_score": 91.5589091152468,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 91.5589091152468,
      "score_before_time_limit": 91.5589091152468,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.07633873149101816,
      "ece": 0.04895343837487054,
      "n": 375,
      "score": 90.20931232502589,
      "score_before_time_limit": 90.20931232502589,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9253333333333333,
      "interval90_width": 0.2215555555555555,
      "n": 375,
      "nrps": 0.06417626744713817,
      "score": 88.84365404499025,
      "score_before_time_limit": 88.84365404499025,
      "time_limit_failed_share": 0,
      "top_ece": 0.07947532582652832
     }
    },
    "score": 90.20395849508765
   },
   "capability": 82.03378630704755,
   "code_revision": "THE AUTHOR'S OWN SERVING MODULES, run unmodified: DOPEST-studio/dopest-model at 027d9d6ae2c66c9ac1bf485c6c68c9d06f887958, six vendored Python modules under dopest_model/ (serve.py sha256 349e64849ef99346edeb9a75fa4c0fe3711cb77d407422766b32d3262ce8d7d3, 174 lines; torch_backend.py 69d6d2127f1cd67ad3e083c732f960bc051b0b1992c79079d1f52564e66d4430, 443 lines; prompt.py 87501fb5b8411852f1a489079038ae8da5eb17913bd60b4d175f5d2e726447db, 55 lines; calibrate.py 75d0f6e15fafd5bda96b3b8fae1fcdc8ba6f727e39779b3a469436dbeb9451be, 59 lines; data.py 377902f317605c0907ba7e4110452494a7b40d8f5c1822f9b89ccc2c64893cf4, 43 lines; __init__.py 5124893506e0d3fdd2f68b17c6a8f10d10c5b2c47aa573de53ac00ca66610a26, 1 line) plus the shipped calibration.json (sha256 37375e0fe4f671177415affb9c2535cedc26cc5af051b7aab266df202d871e18). No scripts, blockb, Modal, training or publishing file was staged or executed; the author's generator and publishing tools (a DeepSeek client reading a secret through the 1Password CLI, Modal app, HF publisher) are outside the serving closure and are never imported (startup forbidden-module audit passed: no modal, openai, dopest_model.modal_app, dopest_model.blockb.* or scripts.* in sys.modules). Entrypoint: the author's dopest_model.serve through our guarded runpy launcher (ctx dopest_launcher.py sha256 02e47ca05a772c533f1966a62a82b2fe25a98206e8806a5432c38bce7b62dd63), with explicit local paths and the author's own flags exactly as shipped: --orders 1 --bidir full --host 127.0.0.1 --port 8000 (--orders 1 and --bidir full are the defaults of the pinned serve.py; the model card at the pinned adapter revision instead calls --orders 2 the default; no order averaging, no extra readout, no quantisation). Native endpoint POST /v1/systemone, no response shim; the server reports output_tokens 0 natively. Model: BF16 on one CUDA device, google/gemma-4-31B-it at 842da3794eaa0b77d5f08bae87a17459d91ff475 plus the PEFT LoRA adapter dopest-studio/dopest-1.0-31b at 06e30b476d00cb84c2bd03999d54b63b58b106c1 (r=32, alpha 64, adapter_model.safetensors sha256 32604c59bb2cbbb05c46fc7e3490654dc9713a55502e7e295811a68a6405af95, 261,423,032 bytes). Both the base and the adapter were downloaded by our code at the pinned revisions and hash-checked file by file on the node (12 files, 0 failures) and read-only in the container; HF offline flags set. Dependency environment only: python:3.12.12-slim-bookworm@sha256:2986c55feb36e6cae00fa1fefb454283e4b33f35e75ff8bdd123b134130be301, derived image sha256:eba61083d29c9113c961949e50b53f0af63e2596dc5f8c0ff8b20596f1291c40, built offline (docker build --network none) from hash-pinned wheels (ctx/requirements.lock sha256 49d47883214d995acb80268736430fe6c6b3133c1b25fa21f54eb47d7c20c3e0). Runtime receipt: torch 2.14.1, transformers 5.20.2, peft 0.21.2, accelerate 1.15.0, tokenizers 0.23.3. Source reviewed before measurement; only the six-module serving closure was executed in an offline, credential-free disposable environment. The author's serving code is public on GitHub (DOPEST-studio/dopest-model, created 24 Oct 2026); an earlier version of this note said the repository had become private, which was our error; the measured vendored bytes remain pinned above. No server code is redistributed by this release. Weights are a BF16 base plus adapter, not a merged checkpoint.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 8752.36566075 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=native). The row's own scored cost basis: 814,647 input + 0 output tokens over 1,485 decisions = 548.583838384 tokens/decision. GPU-time cost USD 0.0378930421646261/1,000; old list-price cost USD 0.0493725454545455/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0378930421646261/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05941758
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03789304216462614
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of DOPEST 1.0 (31B) in this intake (our own intake). FINE-TUNE OF GEMMA 4: dopest-studio/dopest-1.0-31b is a PEFT LoRA adapter (r=32, alpha 64) on google/gemma-4-31B-it, served with the author's bidirectional option-readout path (--bidir full). It is a different set of weights from the Turion, KnowLine, Aleria and ajev rows on the same base: the adapter safetensors file shares 0 LFS sha256 with the pinned snapshots of those four repositories (byte-identity evidence only, not proof of independent training). The name is 'Jev-compatible' only: it is a DOPEST model, not a Jev model. AUTHOR'S OWN SERVING MODULES on our rented hardware, run unmodified through our guarded launcher (see code_revision). NATIVE OUTPUTS: author-server probabilities for choice, noul and score from option-letter logits (softcap 30) at the author's shipped per-type calibration temperatures; no generated explanation. CONFIGURATION: --orders 1 --bidir full, BF16, calibration unchanged; --orders 1 is the default of the pinned server code, whose help text warns that --orders 2 adds about 70% more input tokens per decision and falls outside the Jev-class cost cap. The model card at the pinned adapter revision calls --orders 2 the default and reports its own figures with two orders; --orders 2 was not measured and no result for it is inferred. The causal-readout alternative was not measured either. CALIBRATION AXIS NOTE: the shipped calibration.json (temperatures choice 4.709, noul 7.378, score 4.191) was fitted by the author on the PUBLIC JevBench v1.2 items (choice on 139, noul on 74, 213 of the 231 public items by the file's own counts; the score temperature on the author's own held-out split of 627 items). The public split is published for this purpose and nothing was fitted on sealed items, but the Calibration axis for this row is therefore not a held-out figure with respect to the public items; the model card says so itself. JEVBENCH-AWARE DEFAULT: the shipped calibration contains a yes/no commit floor of 0.801 that moves any answer with P(yes) strictly between 0.199 and 0.801 to the floor on its own side (P(yes) 0.801 or 0.199), because JevBench scores such answers as abstentions. It is the author's default, ships in the artifact, and is measured as shipped (frozen with the configuration before measurement); we did not measure a variant without it and infer nothing about one. TRAINING DATA (author statement, not independently verified): declared synthetic, Apache-2.0, options shuffled; about 28.9k of the 53.3k items (block B) were written by DeepSeek V4.1 Flash and kept only when Qwen3.8-27B and gpt-oss-120b agreed, the rest are rule-engine generated; no Jev outputs are declared; checked by 8-gram overlap against OpenDecisions and the PUBLIC JevBench items only (maximum 8% overlap per the author). It was NOT checked against the sealed items, and we have not checked the training data against JevBench items either; contamination of the sealed draw is unchecked. LICENCE NOTE: adapter Apache-2.0 per the model card; the base is public and ungated; only a score and disclosures are published. CONTEXT AND TRUNCATION (NOT REFUSALS): the author's server has a 12,288-token prompt limit. A longer state is cut proportionally by characters, keeping its START and appending ' [...]', and the model still answers. 15 items were answered on such a truncated state: the draw's 15 standard long positions, counted and frozen before launch in the preregistered truncation plan and matched exactly after the run (original over-limit predicate, final prompt at most 12,288 tokens, retained length and changed-evidence hash checked per item). They are TRUNCATION, never refusals: every one returned HTTP 200 with a probability answer, scored as normal answers; their answers rest on a prompt missing the end of the state. ANSWERED: 1,500 of 1,500, HTTP 200 on every row, no refusal, no error. 10 s RULE: 0 decisions over 10 s, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); median 0.236 s, p95 0.436 s, maximum 9.633 s across all 1,500 requests. The slow requests are exactly the 15 truncated items: each took 8.91-9.63 s, so the slowest is 0.37 s below the limit; every other request took at most 0.82 s. These times were measured on this one A100 80 GB and are preserved as measured; no request exceeded the v1.7 ten-second rule, so the rule changes no item. USAGE: the server-native input usage (990,293 tokens) equals our first-party count exactly, 0 mismatches over 1,500 items; output_tokens 0 natively. DETERMINISM: first 50 requests run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. ATTEMPT HISTORY: three attempts, one measured. Attempt 1 stopped with no scored output, most likely because of an evaluator strict-wrapper import-path defect; missing driver logs prevent definitive attribution. Input was staged but no item reached the model. Attempt 2 was stopped before any item was processed for a usage-gate check. Neither attempt produced a sealed item result; the measured third attempt answered all 1,500 rows. These attempts belong to one admitted measurement.",
   "display": "DOPEST 1.0 31B",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-25",
    "release": "v1.7.4",
    "results_sha256": "115e7cfdfeab39c739854f3d24de4252d0ef4e876b25e4d59108d892639ccbe7"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA A100-SXM4-80GB (81,920 MiB, driver 580.178.04), FIN-03 (Finland, EU), PAYG non-spot, USD 2.2311917808219177/h recorded by the runner, 300 GB OS volume. One VM for this row's measured attempt. Measured 25 Oct 2026, VM window about 17:22-18:03Z, about USD 1.53 GPU rental (elapsed time at the recorded hourly rate; attempts 1 and 2 are not included in that figure). Author server at 127.0.0.1:8000 in a --network none --read-only, cap-drop-ALL container started through env -i: read-only code, model and input mounts, tmpfs caches, private output, empty provider credentials (DEEPSEEK, OPENAI, HF, MODAL, OP) and HF offline flags. Before sealed input: in-container network cut proven (only loopback, no IPv4/IPv6 default route, DNS and TCP fail); model files re-hashed (12 files, 0 failures); a synthetic pre-warm passed on the first try; the TTL fit gate admitted the run. Input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b was re-checked inside the container (match). Complete 1,500-row run, rc 0; scored loop 516.0 s. Raw output sha256 33fe1dc0d85a195fe0bd65d1d0a0fb21c9d78abc1d6c72631eceec306bf33ee8 (the pull-hash receipt). Node-shred and local-shred receipts verified; exact-id teardown receipt confirms the VM closed. GPU rental is separate from the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 76.53207563853452,
    "I_sealed": 71.19515259948038,
    "base": 73.86361411900745,
    "excess": -4.502603731301569,
    "gap": 5.336923039054142,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 80.81042171537538,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 76.26895187870798,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 81.82125603864733,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 70,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 66.96454916158085,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.67050962487976,
       "hard": 73.23175672795763,
       "judge": 69.08276243593521,
       "standard": 48.899833885646274
      }
     },
     "sealed|choice": {
      "cc": 79.25685918078398,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 77.10188504683704,
       "judge": 75.31831537708129,
       "standard": 82.91499683611053
      }
     },
     "sealed|noul": {
      "cc": 77.8261573732495,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 93.10344827586208,
       "hard": 69.0909090909091,
       "judge": 81.53846153846153,
       "standard": 82.08955223880596
      }
     },
     "sealed|score": {
      "cc": 56.502441244407684,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 70.60623359744143,
       "hard": 59.45848850035945,
       "judge": 52.21624554491853,
       "standard": 49.96774410522102
      }
     }
    }
   },
   "jevbench_score": 71.65979612895147,
   "key": "dopest-1.0-31b",
   "last_measured_on": "2026-10-25",
   "licence": "Apache-2.0 adapter per the model card; base google/gemma-4-31B-it public, ungated; author serving code public on GitHub at the pin (no serving code redistributed here)",
   "listing": "ranked",
   "measured_in": "v1.7.4",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "dopest-studio/dopest-1.0-31b@06e30b476d00cb84c2bd03999d54b63b58b106c1 + google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475",
   "new_in": "v1.7.4",
   "noul_decisive": {
    "acc_among_decisive": 0.9013333333333333,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 62,
   "ranked": true,
   "ranks": {
    "A": 62,
    "B": 37,
    "C": 48,
    "capability": 19
   },
   "repo": "https://huggingface.co/dopest-studio/dopest-1.0-31b",
   "scores": {
    "A": 71.65979612895147,
    "B": 72.08997596193838,
    "C": 71.65979612895147
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.6221003080001083,
    "p50_s_raw": 0.23605015400005414,
    "p95_s_adjusted": 1.0204241434998946,
    "p95_s_raw": 0.43521207174994736
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 72.42861969453496,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 42.28585908360487
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 72.42861969453496
      },
      "calibration_gen": {
       "accuracy": 0.900990099009901,
       "cc": 83.55689784600771,
       "cc_by_type": {
        "choice": 90.45599151643691,
        "noul": 92,
        "score": 68.21470202158623
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 83.55689784600771
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 85.42509962738393,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 100,
        "score": 66.03139644312739
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.42509962738393
      },
      "fairness": {
       "accuracy": 1,
       "cc": 97.32485084367364,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 91.97455253102096
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 97.32485084367364
      },
      "jaggedness": {
       "accuracy": 0.6666666666666666,
       "cc": 57.52984750473081,
       "cc_by_type": {
        "choice": 52.358036573628496,
        "noul": 83.78378378378379,
        "score": 36.44772215678016
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 57.52984750473081
      },
      "llm_router": {
       "accuracy": 0.9009584664536742,
       "cc": 80.83201403535305,
       "cc_by_type": {
        "choice": 91.28290921898389,
        "noul": 89.69072164948453,
        "score": 61.522411237590745
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 80.83201403535305
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -50.072698873773135,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100,
        "score": -46.99229016970649
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.8148148148148148,
       "cc": 68.58361993242174,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 100,
        "score": 55.23539587973942
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 68.58361993242174
      },
      "multilingual": {
       "accuracy": 0.8145454545454546,
       "cc": 69.98283233983976,
       "cc_by_type": {
        "choice": 83.37137267688294,
        "noul": 61.111111111111114,
        "score": 65.46601323152521
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.98283233983976
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 76.86286101262225,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 80,
        "score": 57.25524970453342
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 76.86286101262225
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 86.03629844560731,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 73.12391035183698
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 86.03629844560731
      },
      "usecase": {
       "accuracy": 0.740909090909091,
       "cc": 63.636674525057174,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 62.5,
        "score": 54.14531769281857
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 63.636674525057174
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 41.55133520505572,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": 31.796862758024304
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 41.55133520505572
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 82.70373194683133,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 48.11119584049397
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 82.70373194683133
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 90.77499260059749,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 100,
        "score": 80.28957957170395
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 90.77499260059749
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.6329026255702,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.7515942826696
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.6329026255702
      },
      "en": {
       "accuracy": 0.8297132927888793,
       "cc": 74.42207277896841,
       "cc_by_type": {
        "choice": 80.01000720030265,
        "noul": 83.45323741007194,
        "score": 59.80297372653064
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 74.42207277896841
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 59.822549681390264,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": -0.9564945121097956
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 59.822549681390264
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.51185458407953,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 98.98911566480692
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.51185458407953
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 95.6297668151202,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 86.88930044536059
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 95.6297668151202
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 78.83452155500048,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100,
        "score": 53.26334120131428
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 78.83452155500048
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 66.10076392857052,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -1.6977082142884514
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 66.10076392857052
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 51.46005379987258,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 60.00000000000001,
        "score": 8.677222559268237
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 51.46005379987258
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 89.5432854621428,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 68.62985638642843
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 89.5432854621428
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.62476098933989,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.54094963468636
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.62476098933989
      },
      "nl": {
       "accuracy": 0.8,
       "cc": 62.86933662121752,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 55.27467653031923
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 62.86933662121752
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 69.29504590369618,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 60.00000000000001,
        "score": 63.12000863787652
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 69.29504590369618
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 75.07031566374961,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 57.03180733302842
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 75.07031566374961
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 53.14113469823439,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 50.48723388193721
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 53.14113469823439
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 47.9208607230672,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 43.7625821692016
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 47.9208607230672
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 84.94098195369271,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 87.08101037720719
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 84.94098195369271
      },
      "zh": {
       "accuracy": 1,
       "cc": 84.93108398002427,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 54.79325194007283
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 84.93108398002427
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8493333333333334,
       "cc": 80.53923726056756,
       "cc_by_type": {
        "choice": 80.53923726056756
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 80.53923726056756
      },
      "noul": {
       "accuracy": 0.9013333333333333,
       "cc": 80.26666666666667,
       "cc_by_type": {
        "noul": 80.26666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 80.26666666666667
      },
      "score": {
       "accuracy": 0.7066666666666667,
       "cc": 59.42746564835923,
       "cc_by_type": {
        "score": 59.42746564835923
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 59.42746564835923
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 41.05897538785715,
       "cc_by_type": {
        "choice": 68,
        "noul": 25,
        "score": 30.176926163571473
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 41.05897538785715
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 82.04418817627156,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 46.13256452881467
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 82.04418817627156
      },
      "customer_support": {
       "accuracy": 0.9285714285714286,
       "cc": 78.46067484367381,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 35.38202453102145
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 78.46067484367381
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -4.516858930586203,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -33.333333333333336,
        "score": 1.6541015708144924
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 62.93179903221134,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 69.95481738648913
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 62.93179903221134
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 10.16173871318936,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 66.66666666666667,
        "score": -85.86698511829356
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 10.16173871318936
      },
      "financial_crime": {
       "accuracy": 0.7,
       "cc": 64.87957233303455,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 71.42857142857142,
        "score": 42.817988707787094
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 64.87957233303455
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 46.420336069025474,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0,
        "score": 70.61989322449801
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 46.420336069025474
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 36.81828378885919,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": 60.00000000000001,
        "score": 54.62151803324422
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 36.81828378885919
      },
      "knowledge_graphs": {
       "accuracy": 0.6818181818181818,
       "cc": 68.63444360920249,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100,
        "score": 44.447228043881594
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 68.63444360920249
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 64.53136213232108,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 15.816308619185449
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 64.53136213232108
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 57.49007644249298,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 36.986358359737004
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 57.49007644249298
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 88.09901957317152,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 64.29705871951455
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 88.09901957317152
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 81.8772643950808,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 100,
        "score": 50.30711786056708
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.8772643950808
      },
      "moderation_trust_safety": {
       "accuracy": 0.8888888888888888,
       "cc": 59.72881734791644,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0,
        "score": 90.70276682301994
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 59.72881734791644
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 65.82583998474513,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 64.14418662090208
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 65.82583998474513
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 45.43602460691172,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50,
        "score": 15.86472406704058
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 45.43602460691172
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 42.816239018492716,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 0,
        "score": 57.43422430185494
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 42.816239018492716
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.77933763977673,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.33801291933018
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.77933763977673
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.00808202683888,
      91.601122748711
     ],
     "I": [
      69.32489355266667,
      77.46138511809254
     ],
     "capability": [
      78.49838219663346,
      84.08521335573798
     ],
     "composite": null,
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.20395849508765,
     "capability": 82.03378630704755,
     "intelligence": 73.86361411900745
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8493333333333334,
      "calibration": 91.5589091152468,
      "cc": 80.53923726056756,
      "cc_by_type": {
       "choice": 80.53923726056756
      },
      "cc_tiered": 80.03364044807968,
      "cc_tiered_open": 80.81042171537538,
      "cc_tiered_sealed": 79.25685918078398,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 80.53923726056756
     },
     "noul": {
      "accuracy": 0.9013333333333333,
      "calibration": 90.20931232502589,
      "cc": 80.26666666666667,
      "cc_by_type": {
       "noul": 80.26666666666667
      },
      "cc_tiered": 79.82370670594841,
      "cc_tiered_open": 81.82125603864733,
      "cc_tiered_sealed": 77.8261573732495,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 80.26666666666667
     },
     "score": {
      "accuracy": 0.7066666666666667,
      "calibration": 88.84365404499025,
      "cc": 59.42746564835923,
      "cc_by_type": {
       "score": 59.42746564835923
      },
      "cc_tiered": 61.73349520299426,
      "cc_tiered_open": 66.96454916158085,
      "cc_tiered_sealed": 56.502441244407684,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 59.42746564835923
     }
    },
    "raw_on_A": {
     "calibration": 86.2575143231718,
     "capability": 79.52655727008587,
     "intelligence": 72.79560021699996
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 67.06410826293356,
    "Emphasis on Accuracy 60:20:20": 69.62794500701038,
    "Emphasis on Cost 20:20:60": 60.441281677361886,
    "Emphasis on Speed 20:60:20": 72.32601422148595,
    "Intelligence only": 73.86361411900745
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Nace AI",
   "axes": {
    "calibration": 89.56795021085651,
    "cost": 55.336763284582396,
    "intelligence": 78.89021516142793,
    "speed": 89.64251650815261
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0614702656186141,
      "ece_score": 87.70594687627718,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.70594687627718,
      "score_before_time_limit": 87.70594687627718,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.0668449197860963,
      "brier": 0.06226407500935928,
      "ece": 0.05292380823378494,
      "n": 374,
      "score": 89.41523835324301,
      "score_before_time_limit": 89.41523835324301,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.946524064171123,
      "interval90_width": 0.13235294117647062,
      "n": 374,
      "nrps": 0.04098278578392381,
      "score": 91.58266540304936,
      "score_before_time_limit": 91.58266540304936,
      "time_limit_failed_share": 0,
      "top_ece": 0.06368195307754448
     }
    },
    "score": 89.56795021085651
   },
   "capability": 84.22908268614222,
   "composite_ci95": {
    "A": [
     73.86656464052277,
     76.21494021691677
    ],
    "B": [
     74.05703618344492,
     77.33666212419148
    ],
    "C": [
     73.86656464052277,
     76.21494021691677
    ]
   },
   "cost": {
    "basis": "Launch list price: Drex Max public launch list tariff USD 0.06 per million input tokens, output not billed, https://drex.nace.ai/docs/drex/reference/pricing (saved copy sha256 95d1c429, fetched 21 Oct 2026 20:18 UTC; fetched again 22 Oct 2026 01:12 UTC with identical bytes). Counted as an operator standard launch list price by the frozen pricing module pricing_v15 (sha256 a8a05890ddab2d39d1a1bd543f2b5c83842bdc70aa54766788bc89b7a8a448d7), which now carries a launch-list-price line for this product. Applied to provider-reported usage; not a base-model estimate. COMMON COST BASIS (v1.6.30): common 1,485 items, excluding the 15 longest for every row; 762,693 input / 262,045 output tokens. All-item usage 1,942,633 input / 264,615 output over 1,500 items.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.07770532
    },
    "kind": "listed",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.030815878787878788
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Drex Max (Nace AI)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid public fast-lane; full S1200+P300 draw v1.6-regular-20261012, unchanged evaluator API runner from Sandy, customer API key; full-set exposure gate logged all sealed items before the first scored request. Native probabilities; latency has no API adjustment.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 81.66095761967301,
    "I_sealed": 76.11947270318284,
    "base": 78.89021516142793,
    "excess": -4.298041853865534,
    "gap": 5.541484916490177,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 85.06220262367293,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 90.56307014784524,
       "judge": 84.17930125247199,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 85.29951690821257,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 70,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 74.62115332713357,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.894225726887605,
       "hard": 77.86140310269487,
       "judge": 79.87410650396917,
       "standard": 67.12468781088054
      }
     },
     "sealed|choice": {
      "cc": 83.42288824648435,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 80.2243552677229,
       "judge": 80.60724779627816,
       "standard": 87.66083104830204
      }
     },
     "sealed|noul": {
      "cc": 70.73090779524131,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 60.00000000000001,
       "judge": 72.30769230769232,
       "standard": 82.08955223880596
      }
     },
     "sealed|score": {
      "cc": 74.20462206782285,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 83.85760877630031,
       "hard": 76.79135886109052,
       "judge": 72.01874094079821,
       "standard": 67.48347681758575
      }
     }
    }
   },
   "jevbench_score": 75.37621071625281,
   "key": "drex-max",
   "last_measured_on": "2026-10-21",
   "licence": "Proprietary vendor-hosted API; weights and serving code not independently inspected",
   "listing": "ranked",
   "measured_in": "v1.6.38",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:https://drex.nace.ai#drex-max",
   "noul_decisive": {
    "acc_among_decisive": 0.9426934097421203,
    "decisive": 349,
    "decisive_rate": 0.9331550802139037,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 6,
    "axes": {
     "calibration": 89.56795021085651,
     "cost": 55.336763284582396,
     "intelligence": 78.89021516142793,
     "speed": 89.80651018227358
    },
    "board_release": "v1.6.43",
    "capability": 84.22908268614222,
    "capability_api_board_rank": 3,
    "capability_open_board_rank": null,
    "capability_rank": 5,
    "composite_ci95": {
     "A": [
      73.89436205802828,
      76.24453356276406
     ],
     "B": [
      74.07938728544397,
      77.3610370295492
     ],
     "C": [
      73.89436205802828,
      76.24453356276406
     ]
    },
    "jevbench_score": 75.40515618514833,
    "measured_in": "v1.6.38",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 7,
    "ranks": {
     "A": 7,
     "B": 3,
     "C": 7,
     "capability": 10
    },
    "scores": {
     "A": 75.40515618514833,
     "B": 76.0773154141038,
     "C": 75.40515618514833
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.2520033140317537,
     "p50_s_raw": 0.2520033140317537,
     "p95_s_adjusted": 0.41489932631957344,
     "p95_s_raw": 0.41489932631957344
    },
    "views": {
     "Balanced 33:33:33": 71.62971034982631,
     "Emphasis on Accuracy 60:20:20": 74.3674134322497,
     "Emphasis on Cost 20:20:60": 64.08251746798757,
     "Emphasis on Speed 20:60:20": 77.93969491847153,
     "Intelligence only": 78.89021516142793
    }
   },
   "rank": 15,
   "ranked": true,
   "ranks": {
    "A": 15,
    "B": 6,
    "C": 14,
    "capability": 13
   },
   "repo": "https://drex.nace.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 75.37621071625281,
    "B": 76.05374262335641,
    "C": 75.37621071625281
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2538145645521581,
    "p50_s_raw": 0.2538145645521581,
    "p95_s_adjusted": 0.42779116374440485,
    "p95_s_raw": 0.42779116374440485
   },
   "status": {
    "answered_ok": 1498,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API through the unchanged TypeSafe adapter; no vendor code ran locally. 2 HTTP 529 provider-overloaded failures retained and scored as failures. Public JevBench items were used to pick the checkpoint; the public set is treated as seen during selection. Customer statements about serving and training are unverified. The vendor has since updated the model behind this API id; this result is from 2026-10-21.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 87.32730011427283,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 70.31523367615182
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 87.32730011427283
      },
      "calibration_gen": {
       "accuracy": 0.7722772277227723,
       "cc": 66.4168117636221,
       "cc_by_type": {
        "choice": 65.00530222693531,
        "noul": 60.00000000000001,
        "score": 74.245133063931
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 66.4168117636221
      },
      "composite_gen": {
       "accuracy": 0.9306930693069307,
       "cc": 90.98786119357764,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 83.62302350846554
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 90.98786119357764
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.9811260569162,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.94337817074862
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.9811260569162
      },
      "jaggedness": {
       "accuracy": 0.7611111111111111,
       "cc": 62.67781187887625,
       "cc_by_type": {
        "choice": 63.90760346487007,
        "noul": 45.945945945945944,
        "score": 78.17988622581275
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 62.67781187887625
      },
      "llm_router": {
       "accuracy": 0.9265175718849841,
       "cc": 83.03900487815365,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 93.81443298969072,
        "score": 60.84982123268957
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.03900487815365
      },
      "long_state": {
       "accuracy": 0.5333333333333333,
       "cc": -7.819371042052626,
       "cc_by_type": {
        "choice": 61.29032258064516,
        "noul": 0,
        "score": -84.74843570680304
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.8148148148148148,
       "cc": 67.45921990260773,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 50,
        "score": 77.11992774906025
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 67.45921990260773
      },
      "multilingual": {
       "accuracy": 0.8436363636363636,
       "cc": 76.08800384521719,
       "cc_by_type": {
        "choice": 81.41506358004565,
        "noul": 72.22222222222223,
        "score": 74.6267257333837
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 76.08800384521719
      },
      "negation_pair_gen": {
       "accuracy": 0.8837209302325582,
       "cc": 74.73255877300765,
       "cc_by_type": {
        "choice": 96.66666666666667,
        "noul": 66.66666666666667,
        "score": 60.86434298568961
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.73255877300765
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 83.67877565230252,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 66.0513419719226
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 83.67877565230252
      },
      "usecase": {
       "accuracy": 0.8454545454545455,
       "cc": 73.95953326890493,
       "cc_by_type": {
        "choice": 82.84313725490196,
        "noul": 62.5,
        "score": 76.53546255181287
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 73.95953326890493
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 69.91186881655202,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50,
        "score": 78.7832254972751
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 69.91186881655202
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 58.44437051865251,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 41.99977822262423
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 58.44437051865251
      },
      "de": {
       "accuracy": 0.875,
       "cc": 87.69742394292284,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 86.98607713850303
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.69742394292284
      },
      "el": {
       "accuracy": 0.9333333333333333,
       "cc": 94.41307136986232,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100,
        "score": 99.9989906458998
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 94.41307136986232
      },
      "en": {
       "accuracy": 0.8705473501303215,
       "cc": 78.87188263043777,
       "cc_by_type": {
        "choice": 85.28209321340964,
        "noul": 75.53956834532374,
        "score": 75.7939863325799
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 78.87188263043777
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 72.33983928560271,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 26.80744607866782
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 72.33983928560271
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.76471453815516,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.96081028113215
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.76471453815516
      },
      "fr": {
       "accuracy": 1,
       "cc": 99.0532296631748,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.15968898952437
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 99.0532296631748
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 70.61814713883847,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 75,
        "score": 53.61421795282821
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 70.61814713883847
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 75.83799074906126,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 41.54359032980268
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 75.83799074906126
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 54.328167608501246,
       "cc_by_type": {
        "choice": 100,
        "noul": 39.99999999999999,
        "score": 22.98450282550375
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 54.328167608501246
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 92.58033340509049,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 77.74100021527148
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 92.58033340509049
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.7683454076434,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.9717028895969
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.7683454076434
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 17.73905043083302,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": -28.168130092782317
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 17.73905043083302
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8717948717948718,
       "cc": 81.50650313356748,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80,
        "score": 79.7543803274904
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 81.50650313356748
      },
      "pt": {
       "accuracy": 0.8928571428571429,
       "cc": 84.51664635695569,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100,
        "score": 74.76384596538684
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 84.51664635695569
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 47.512402499217224,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 60.00000000000001,
        "score": 59.13295217850272
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 47.512402499217224
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 82.40769768899798,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 68.96222350177659
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 82.40769768899798
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 83.90750936589232,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 83.98059261380598
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 83.90750936589232
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 57.55187446448804,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 39.32229006013077
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 57.55187446448804
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8813333333333333,
       "cc": 84.67249660345587,
       "cc_by_type": {
        "choice": 84.67249660345587
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.67249660345587
      },
      "noul": {
       "accuracy": 0.8773333333333333,
       "cc": 75.46666666666665,
       "cc_by_type": {
        "noul": 75.46666666666665
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 75.46666666666665
      },
      "score": {
       "accuracy": 0.8186666666666667,
       "cc": 74.40265595252149,
       "cc_by_type": {
        "score": 74.40265595252149
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 74.40265595252149
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 40.94013002702126,
       "cc_by_type": {
        "choice": 84,
        "noul": -25,
        "score": 63.82039008106377
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 40.94013002702126
      },
      "code_linting": {
       "accuracy": 0.9583333333333334,
       "cc": 94.74649038827987,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 84.23947116483957
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 94.74649038827987
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 56.18418746315232,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 100,
        "score": 33.59321279596102
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 56.18418746315232
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 10.678347318840489,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 33.33333333333333,
        "score": -42.81875921306917
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 10.678347318840489
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 65.8121592702109,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 93.08865172367618
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 65.8121592702109
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 63.11469221028199,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": 81.16797600191516
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 63.11469221028199
      },
      "financial_crime": {
       "accuracy": 0.8666666666666667,
       "cc": 69.85403609350806,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 42.85714285714286,
        "score": 76.50888699200877
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 69.85403609350806
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 62.51172431161607,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 50,
        "score": 68.89405795226982
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 62.51172431161607
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 58.5760942817511,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 78.22828284525329
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 58.5760942817511
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 78.42207904152188,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100,
        "score": 48.114202863323655
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 78.42207904152188
      },
      "lead_generation": {
       "accuracy": 0.88,
       "cc": 79.80323774108199,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 61.63193544546821
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 79.80323774108199
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 26.355654431569025,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 42.50782350976085
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 26.355654431569025
      },
      "llm_guardrails": {
       "accuracy": 1,
       "cc": 96.94611245850201,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 90.83833737550606
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 96.94611245850201
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 81.10660381485808,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100,
        "score": 49.16396728873009
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.10660381485808
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 84.70157923454258,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100,
        "score": 77.13736726216898
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 84.70157923454258
      },
      "recruiting": {
       "accuracy": 0.8888888888888888,
       "cc": 86.1084876352515,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100,
        "score": 75.0385548277601
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 86.1084876352515
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 51.609571747494265,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 50,
        "score": 19.6070403656355
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 51.609571747494265
      },
      "scientific_discovery": {
       "accuracy": 0.8695652173913043,
       "cc": 69.05169114747838,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": 33.33333333333333,
        "score": 88.3144937322902
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 69.05169114747838
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.97852779958207,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.9355833987462
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.97852779958207
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.5532389091344,
      91.2233312473974
     ],
     "I": [
      74.71372483564313,
      82.2819224684923
     ],
     "capability": [
      80.68052980412993,
      86.222957448835
     ],
     "composite": {
      "A": [
       73.86656464052277,
       76.21494021691677
      ],
      "B": [
       74.05703618344492,
       77.33666212419148
      ],
      "C": [
       73.86656464052277,
       76.21494021691677
      ]
     },
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.56795021085651,
     "capability": 84.22908268614222,
     "intelligence": 78.89021516142793
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8813333333333333,
      "calibration": 87.70594687627718,
      "cc": 84.67249660345587,
      "cc_by_type": {
       "choice": 84.67249660345587
      },
      "cc_tiered": 84.24254543507864,
      "cc_tiered_open": 85.06220262367293,
      "cc_tiered_sealed": 83.42288824648435,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.67249660345587
     },
     "noul": {
      "accuracy": 0.8773333333333333,
      "calibration": 89.41523835324301,
      "cc": 75.46666666666665,
      "cc_by_type": {
       "noul": 75.46666666666665
      },
      "cc_tiered": 78.01521235172694,
      "cc_tiered_open": 85.29951690821257,
      "cc_tiered_sealed": 70.73090779524131,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 75.46666666666665
     },
     "score": {
      "accuracy": 0.8186666666666667,
      "calibration": 91.58266540304936,
      "cc": 74.40265595252149,
      "cc_by_type": {
       "score": 74.40265595252149
      },
      "cc_tiered": 74.41288769747821,
      "cc_tiered_open": 74.62115332713357,
      "cc_tiered_sealed": 74.20462206782285,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 74.40265595252149
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 71.59488791056847,
    "Emphasis on Accuracy 60:20:20": 74.3448882133178,
    "Emphasis on Cost 20:20:60": 64.06579112533402,
    "Emphasis on Speed 20:60:20": 77.86551943721958,
    "Intelligence only": 78.89021516142793
   }
  },
  {
   "adapter_id": "images/eikos-27b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 89.97714437568723,
    "cost": 56.05951697168831,
    "intelligence": 64.40270761245412,
    "speed": 87.88773525505371
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03950772533975903,
      "ece_score": 92.09845493204818,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 92.09845493204818,
      "score_before_time_limit": 92.09845493204818,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.19137466307277629,
      "brier": 0.05632135167655414,
      "ece": 0.06181891259769947,
      "n": 371,
      "score": 87.6362174804601,
      "score_before_time_limit": 87.6362174804601,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.964769647696477,
      "interval90_width": 0.3172990063233966,
      "n": 369,
      "nrps": 0.05868305372968562,
      "score": 90.19676071455342,
      "score_before_time_limit": 90.19676071455342,
      "time_limit_failed_share": 0,
      "top_ece": 0.06869086598962304
     }
    },
    "score": 89.97714437568723
   },
   "capability": 77.18992599407068,
   "class": "unclassified",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 921,181 input + 0 output tokens over 1,624 decisions = 567.229679803 tokens/decision. GPU-time cost USD 0.0291529674422963/1,000; old list-price cost USD 0.238236465517241/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0291529674422963/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.029152967442296302
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Eikos-27B (caiovicentino1, Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 65.62981582129679,
    "I_sealed": 63.17559940361144,
    "base": 64.40270761245412,
    "excess": -0.1191712247528991,
    "gap": 2.4542164176853447,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 80.41823040534584,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 77.35136835482857,
       "judge": 76.26895187870797,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 58.24775707384403,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 30.000000000000004,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 58.2234599847005,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.643002416487,
       "hard": 61.32359173476558,
       "judge": 58.76732285181849,
       "standard": 49.49763096800006
      }
     },
     "sealed|choice": {
      "cc": 78.56119802347493,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 76.05498484960461,
       "hard": 74.5475113122172,
       "judge": 78.58472998137803,
       "standard": 87.80638009607095
      }
     },
     "sealed|noul": {
      "cc": 53.35348874031106,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 37.86407766990292,
       "judge": 57.89473684210527,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 57.61211144704836,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 67.33566285144035,
       "hard": 49.71122434442663,
       "judge": 64.06787863213144,
       "standard": 58.86845917247121
      }
     }
    }
   },
   "jevbench_score": 71.61070910537894,
   "key": "eikos-27b",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.98,
    "decisive_rate": 0.8086253369272237,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.97714437568723,
     "cost": 28.689752882469676,
     "intelligence": 64.40270761245412,
     "speed": 88.08630815059152
    },
    "board_release": "v1.6.43",
    "capability": 77.18992599407068,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      17.720899810915583,
      18.261473654252256
     ],
     "B": [
      18.087561450267057,
      18.993796215955186
     ],
     "C": [
      17.656534367876468,
      18.261473654252256
     ]
    },
    "jevbench_score": 18.077889022432696,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 106,
    "rank": 141,
    "ranks": {
     "A": 141,
     "B": 134,
     "C": 123,
     "capability": 37
    },
    "scores": {
     "A": 18.077889022432696,
     "B": 18.627129011951496,
     "C": 18.077889022432696
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.37523653320968153,
     "p50_s_raw": 0.11261826660484076,
     "p95_s_adjusted": 0.4140607553534209,
     "p95_s_raw": 0.13203037767671047
    },
    "views": {
     "Balanced 33:33:33": 15.999272744012083,
     "Emphasis on Accuracy 60:20:20": 17.74116866920125,
     "Emphasis on Cost 20:20:60": 12.523731672975002,
     "Emphasis on Speed 20:60:20": 19.495440346386523,
     "Intelligence only": 21.20399692738122
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 63,
   "ranked": true,
   "ranks": {
    "A": 63,
    "B": 52,
    "C": 49,
    "capability": 50
   },
   "repo": "https://huggingface.co/caiovicentino1/eikos-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.61070910537894,
    "B": 70.04285877930865,
    "C": 71.61070910537894
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.38070584982633593,
    "p50_s_raw": 0.11535292491316795,
    "p95_s_adjusted": 0.4272055875975638,
    "p95_s_raw": 0.1386027937987819
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.83309061783127,
      90.19679304741126
     ],
     "I": [
      59.88680163370944,
      68.49583860207214
     ],
     "capability": [
      73.72374153392347,
      78.55486844196058
     ],
     "composite": null,
     "seed": 36
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.97714437568723,
     "capability": 77.18992599407068,
     "intelligence": 64.40270761245412
    },
    "per_type": {
     "choice": {
      "accuracy": 0.836,
      "calibration": 92.09845493204818,
      "cc": 79.02674232448868,
      "cc_by_type": {
       "choice": 79.02674232448868
      },
      "cc_tiered": 79.48971421441038,
      "cc_tiered_open": 80.41823040534584,
      "cc_tiered_sealed": 78.56119802347493,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 79.02674232448868
     },
     "noul": {
      "accuracy": 0.784,
      "calibration": 87.6362174804601,
      "cc": 56.800000000000004,
      "cc_by_type": {
       "noul": 56.800000000000004
      },
      "cc_tiered": 55.80062290707754,
      "cc_tiered_open": 58.24775707384403,
      "cc_tiered_sealed": 53.35348874031106,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 56.800000000000004
     },
     "score": {
      "accuracy": 0.6853333333333333,
      "calibration": 90.19676071455342,
      "cc": 57.48579408299472,
      "cc_by_type": {
       "score": 57.48579408299472
      },
      "cc_tiered": 57.91778571587443,
      "cc_tiered_open": 58.2234599847005,
      "cc_tiered_sealed": 57.61211144704836,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 57.48579408299472
     }
    },
    "raw_on_A": {
     "calibration": 86.77546118929445,
     "capability": 77.43357682556581,
     "intelligence": 68.09169246183717
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 67.04864466572103,
    "Emphasis on Accuracy 60:20:20": 65.9646021372653,
    "Emphasis on Cost 20:20:60": 62.17358672299847,
    "Emphasis on Speed 20:60:20": 74.07414265913837,
    "Intelligence only": 64.40270761245412
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "baselabs (Base Labs at Baseten; baselabs on Hugging Face)",
   "axes": {
    "calibration": 84.65640909439074,
    "cost": 68.07169708446307,
    "intelligence": 55.83724375498836,
    "speed": 89.7330689398618
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10618694754098373,
      "ece_score": 78.76261049180326,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 78.76261049180326,
      "score_before_time_limit": 78.76261049180326,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.21715817694369977,
      "brier": 0.11100565751402397,
      "ece": 0.05619157372654143,
      "n": 373,
      "score": 88.76168525469171,
      "score_before_time_limit": 88.76168525469171,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9354838709677419,
      "interval90_width": 0.3230286738351253,
      "n": 372,
      "nrps": 0.07802430475032547,
      "score": 86.44493153667727,
      "score_before_time_limit": 86.44493153667727,
      "time_limit_failed_share": 0,
      "top_ece": 0.09653853225806448
     }
    },
    "score": 84.65640909439074
   },
   "capability": 70.24682642468954,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 777,323 input + 1,485 output tokens over 1,485 decisions = 524.44983165 tokens/decision. GPU-time cost USD 0.0115951604823608/1,000; old list-price cost USD 0.0524949831649832/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0115951604823608/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.051970033333333325
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011595160482360806
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "baselabs elephant-1 (9B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h. The launcher and engine run in a docker container with network none (cut proven from inside before any item hash or model load: loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, read-only model and input, empty credential variables, HF offline flags, one request at a time, timed over loopback on the VM. Peak CUDA memory allocated after the scored run 22,898 MiB of 97,887 MiB. Measured 22 Oct 2026 (row 10:27-10:32Z, scored run 198 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720..., checked on the node and inside the container). This is an elephant-only retry on a fresh VM: the first attempt's elephant row, planned on the mammoth-1 VM, failed before any sealed byte was read (the prepared stack image had been pruned by the mammoth cleanup). On this VM the input was shredded from the node and its absence verified, and the VM was removed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 61.08649662617839,
    "I_sealed": 50.58799088379833,
    "base": 55.83724375498836,
    "excess": 0.6589789720243502,
    "gap": 10.498505742380061,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 73.67205398217818,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 66.02705253224286,
       "judge": 80.22412656558998,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 51.134575569358184,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 65.21739130434783,
       "judge": 19.999999999999996,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 58.45286032699879,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 69.72297878787879,
       "hard": 60.82901203703703,
       "judge": 57.210411864406765,
       "standard": 49.92917037037038
      }
     },
     "sealed|choice": {
      "cc": 67.82247430702198,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 82.84625158831004,
       "hard": 59.407887128483864,
       "judge": 66.50342801175319,
       "standard": 79.11832946635731
      }
     },
     "sealed|noul": {
      "cc": 37.396837850775064,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 48.27586206896552,
       "hard": 27.27272727272727,
       "judge": 35.384615384615394,
       "standard": 55.223880597014926
      }
     },
     "sealed|score": {
      "cc": 46.544660493597945,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 70.84345,
       "hard": 39.7783152112676,
       "judge": 45.315041666666666,
       "standard": 49.77238454545453
      }
     }
    }
   },
   "jevbench_score": 71.99915829082904,
   "key": "elephant-1",
   "last_measured_on": "2026-10-22",
   "licence": "MIT (model card metadata since 21 Oct 2026; no LICENSE file; base Qwen/Qwen3.5-9B is Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "baselabs/elephant-1@c54a55c2d362a7e30ff30814d28e39054e954d58",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.910958904109589,
    "decisive": 292,
    "decisive_rate": 0.7828418230563002,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 57,
   "ranked": true,
   "ranks": {
    "A": 57,
    "B": 70,
    "C": 81,
    "capability": 97
   },
   "repo": "https://huggingface.co/baselabs/elephant-1",
   "scores": {
    "A": 71.99915829082904,
    "B": 68.05925107047632,
    "C": 62.355226836692744
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3007341589999669,
    "p50_s_raw": 0.07536707949998345,
    "p95_s_adjusted": 0.35359844430012116,
    "p95_s_raw": 0.10179922215006058
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The authors' own serving runtime: OneJev qev package at 81ce62f1597c91e46767d4d02d6ac2e18534fe94 plus the authors' serve.patch (sha256 f234c125f1d38d82104b9e1f24e75bde51e4401928acfba2fbf5c59ef2fb46b8; 15 qev/ files, manifest sha256 818ca0fc1d26ec6a2369aac210fb1de41d908a9e52b45e25bc6150714f046501), vendored as byte-pinned files (re-verified inside the smoke container: 15 files OK), no git clone, no pip -e, setup.sh never run. The authors' code bundle (engine.py, setup.sh, constraints.txt, serve.patch, readout.json) is byte-identical to the one of baselabs/mammoth-1; only config.json, generation_config.json and the weights differ. It is run through OUR first-party launcher baselabs_launcher.py (sha256 7a0896373a6677034909ffa56f17558cbd9ceaf389a9dafdea59cfc638ce26ed), which builds the authors' qev.engine.DecisionEngine (bfloat16 + fp32 LM head, block-bidirectional attention on the full-attention layers, slot_lm readout, 256 slots, debias 1, no calibration file so T=1, auto fork mode which ran as 'batched', author limits 32,768 / 65,536 tokens) and hands it to OneJev's own create_app on 127.0.0.1:8000. `qev serve` was NOT used: it routes these checkpoints (config has a vision_config) to MMDecisionEngine, which never attaches the block-bidirectional mask; the DecisionEngine is what the authors' own Decision Index runs use. The launcher adds only the authors' own 32BitIndexMath sequential-fork fallback and one read-only /_launcher/info route; no prompt, truncation, calibration, answer or usage change. After the scored run the engine reported noncausal attention with the mask installed and every counted mask build (3,000 of 3,000) carried block ids; the counters show that the mask was built with block ids, not numerical parity with the Decision Index runs or with a causal-mask comparison (neither was run). Code gate: cleared by the benchmark owner (22 Oct 2026 08:42Z); a later addendum covering the attention-mask monkeypatch, the Hub-kernel path and the setup.sh extras was disclosed to the owner afterwards and no fresh confirmation was claimed. DISCLOSED CODE PATTERNS (download/client/kernel paths are inactive; the attention-mask monkeypatch executes offline as described below): setup.sh (git clone, pip install -e, a CUDA extension build that can download a wheel), qev/client.py (urllib HTTP client class, unused), qev/engine.py and mm_engine.py Hub loaders (a local verified snapshot is passed, offline flags set), readout.py torch.load of a head file (checkpoints say slot_lm, no head file), constraints.txt listing wandb and an OpenTelemetry exporter stack (not installed), an in-process monkeypatch of transformers' create_causal_mask to add block ids (readout.py:104-124, runs inside the network-none container), and transformers' optional Hub-kernel path (inactive: the kernels package is not installed). A static source review and two independent runner reviews preceded the run (their one must was closed and re-checked before launch); the later runner changes for this elephant-only retry (keep the stack image) were checked by us (diff and dry-run), not by a further review round. Image: our own build on the pod (network on, before any sealed byte) from pytorch/pytorch:2.8.0-cuda12.8-cudnn9-devel@sha256:a7103283ea7113e10ae5d014bd2342acebda0bc53164b2f7b1dd6eb7a766bdb6 plus 50 hash-pinned wheels (--require-hashes, lock sha256 bf9fa61b272ce83fe5a2ac4c6db219484f929083d46efe0f0a5252ab969c31da, installed with --no-index --no-deps); torch 2.8.0+cu128, transformers 5.19.0, tokenizers 0.23.3, flash-linear-attention 0.5.2; causal-conv1d is not installed (transformers uses its torch fallback); stack image sha256:76a3333125ebdae54475f49994d3e01ea758a0f82c1f85c0ccdf0ebf03e8dbd5, row image sha256:d4f70ee15d359d0c64a093013bf9bab19f4d4e877469c89142df68f39dc00a29. Weights: baselabs/elephant-1 at c54a55c2d362a7e30ff30814d28e39054e954d58 (licence flip commit 5494c13e; weights commit 14537f08, unchanged since), the 8 serving files (18,839,725,537 bytes, serve manifest sha256 c7202744...), fetched by us from the commit-pinned Hub URLs and sha256-verified file by file on the host and again in the smoke container; README, engine.py, setup.sh, constraints.txt, serve.patch and the banner image were not used. FIRST MEASUREMENT (found by our own intake: Decision Index 0.4.1 entry; no outside request for this model). AUTHOR-DEFAULT CONTEXT LIMITS: the engine allows 32,768 tokens for state plus the longest question (strict >) and 65,536 per request and raises an error above that, which the authors' runtime returns through OneJev create_app as HTTP 422, never a truncation; the authors train at 16,384 tokens. The 15 items of 76,374-80,534 tokens were refused exactly as pre-registered and are scored wrong; the largest answered item is 1,854 tokens, inside the trained length. NOT AN AUTHOR-SHIPPED SERVER: the authors ship no server file; our first-party launcher around their DecisionEngine and OneJev's web app is what was timed (see code_revision). SERIAL ONLY: the server handles one request at a time and our driver sent one at a time; determinism over the first 50 positions was bit-identical (50/50, 0 argmax flips). causal-conv1d is not installed, so transformers' slower torch convolution fallback ran. LICENCE: MIT in the card metadata since 21 Oct 2026 (commit 5494c13e changed 'proprietary' to 'mit' and removed the evaluation-only line); there is no LICENSE file, no copyright holder and no carried Qwen attribution, and the Decision Index row still said 'proprietary' on 22 Oct (stale). Weights are identical to the Decision Index-scored checkpoint (14537f08). Two README drafts briefly named Qwen3.5-27B as base model; the pinned card and the config say Qwen3.5-9B. TRAINING DATA: the card gives a mix of 6.21M decision examples (9.5% public decision data) without naming datasets; the Decision Index states that the training includes train splits of Decision Index benchmarks (author-declared), and that its checks found signs that the model's training data overlapped one private new-domain test, which it does not count for this model; we cannot rule out overlap on other tests (our inference); the authors' technical report separately claims zero contamination with the Jev / Artificial Analysis index benchmarks (self-reported, not verified here); we ran no overlap check and no JevBench use or denial is declared, so whether those Decision Index train splits overlap JevBench sources is unknown to us. SELF-REPORTED, unverified: Decision Index 0.4 score 62.65 ('best model under 15B', tied #8 of 208) and an Artificial Analysis replica score of 65.2 on the card; the DI lab's own Decision Index 0.4.1 full score is 60.26 (public 72.24, equal to the claim by the kit rescore; decision agreement with the submitted rows 99.84% on 819 rows against a 98.5% bar), DI-measured latency median 114 ms. 10 s RULE: max measured decision latency 0.30 s (median 0.08 s, p95 0.10 s); 0 decisions over 10 s; in the smoke a 20k-token request took about 1.9 s warm and an exact 32,768-token request about 9.4 s, but no answered item is that large.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 81.48517777777778,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 61.122200000000014
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 81.48517777777778
      },
      "calibration_gen": {
       "accuracy": 0.5841584158415841,
       "cc": 36.321363985728176,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 19.999999999999996,
        "score": 36.684134375
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 36.321363985728176
      },
      "composite_gen": {
       "accuracy": 0.8217821782178217,
       "cc": 67.3911931820814,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 40.74074074074075,
        "score": 67.93690384615384
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 67.3911931820814
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 84.64020925925927,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 93.92062777777778
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.64020925925927
      },
      "jaggedness": {
       "accuracy": 0.5111111111111111,
       "cc": 25.522811215450304,
       "cc_by_type": {
        "choice": 40.808469682386914,
        "noul": -2.7027027027026973,
        "score": 38.46266666666669
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 25.522811215450304
      },
      "llm_router": {
       "accuracy": 0.805111821086262,
       "cc": 69.18685079356193,
       "cc_by_type": {
        "choice": 71.47133926212908,
        "noul": 71.13402061855669,
        "score": 64.95519250000001
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 69.18685079356193
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 33.259192963508426,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 25,
        "score": 11.890980952380946
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 33.259192963508426
      },
      "multilingual": {
       "accuracy": 0.7127272727272728,
       "cc": 53.590397470200806,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": 36.111111111111114,
        "score": 54.982872300469474
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 53.590397470200806
      },
      "negation_pair_gen": {
       "accuracy": 0.7906976744186046,
       "cc": 50.353867777777765,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 39.99999999999999,
        "score": 17.72826999999999
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 50.353867777777765
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 76.1766397089397,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100,
        "score": 55.55694615384616
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 76.1766397089397
      },
      "usecase": {
       "accuracy": 0.6772727272727272,
       "cc": 48.10202756078433,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 25,
        "score": 45.04137680000002
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 48.10202756078433
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 24.111361660561652,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": -39.570676923076945
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 24.111361660561652
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 1,
       "cc": 86.30362000000001,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 58.91086000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 86.30362000000001
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 72.35387820956778,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 75,
        "score": 81.88464347826087
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 72.35387820956778
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.364865642458106,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 99.61415
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.364865642458106
      },
      "en": {
       "accuracy": 0.7141615986099045,
       "cc": 53.212251436303546,
       "cc_by_type": {
        "choice": 69.46583517408865,
        "noul": 40.28776978417265,
        "score": 49.883149350649326
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 53.212251436303546
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.66649451428206,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 30.787411764705887
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.66649451428206
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 50.72039999999999,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 85.49453333333332
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 50.72039999999999
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 76.84436449948402,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 94.06250526315792
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 76.84436449948402
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 70.33820782122905,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 50,
        "score": 77.7744
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 70.33820782122905
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 54.22450476190476,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -37.32648571428572
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 54.22450476190476
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 32.49348811578854,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 39.99999999999999,
        "score": -13.925413333333347
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 32.49348811578854
      },
      "ko": {
       "accuracy": 0.4,
       "cc": 14.908567989417984,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 0,
        "score": 33.614592857142846
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 14.908567989417984
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 92.80816666666665,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 95.09116666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 92.80816666666665
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 44.05853379509379,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 50.79031999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 44.05853379509379
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 61.86896395463657,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 60.84176279069767
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 61.86896395463657
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 72.44503948967501,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 59.762932258064524
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 72.44503948967501
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 37.953801773049655,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 70.45715000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 37.953801773049655
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 54.188839855072466,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": -15.694350000000012
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 54.188839855072466
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.2079,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 61.62370000000001
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.2079
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 10.766112617702452,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -8.94459999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 10.766112617702452
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7653333333333333,
       "cc": 69.68943148548576,
       "cc_by_type": {
        "choice": 69.68943148548576
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 69.68943148548576
      },
      "noul": {
       "accuracy": 0.7093333333333334,
       "cc": 41.866666666666674,
       "cc_by_type": {
        "noul": 41.866666666666674
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 41.866666666666674
      },
      "score": {
       "accuracy": 0.632,
       "cc": 50.03769070397117,
       "cc_by_type": {
        "score": 50.03769070397117
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 50.03769070397117
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 18.88307843137255,
       "cc_by_type": {
        "choice": 68,
        "noul": -50,
        "score": 38.649235294117645
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 18.88307843137255
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 35.477907264957274,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": -26.8996115384615
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 35.477907264957274
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 28.424070731707303,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": -15.947300000000041
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 28.424070731707303
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -1.0775926656920127,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -33.333333333333336,
        "score": -11.419912500000006
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 26.025824849033814,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 54.88906874999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 26.025824849033814
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": -41.56288291404612,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 66.66666666666667,
        "score": -241.04085
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 28.68574496157437,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -14.28571428571429,
        "score": 19.95079230769229
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 28.68574496157437
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 53.46004675554209,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 50,
        "score": 52.19198695652174
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 53.46004675554209
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 35.39760740740741,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 48.69282222222223
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 35.39760740740741
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 65.24514960850475,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 54.76471363636364
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 65.24514960850475
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 52.23264363401455,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 34.99250454545454
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 52.23264363401455
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 12.344700044802865,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -33.333333333333336,
        "score": 34.8835625
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 12.344700044802865
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 59.51451296296296,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 58.54353888888888
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.51451296296296
      },
      "model_routing": {
       "accuracy": 0.8755760368663594,
       "cc": 75.0689772558171,
       "cc_by_type": {
        "choice": 89.48051948051949,
        "noul": 81.81818181818181,
        "score": 53.90823046875
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.0689772558171
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 10.657687181513685,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 66.52200588235296
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 10.657687181513685
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 62.65915098326311,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 71.35721153846156
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 62.65915098326311
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 62.77636666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 38.3291
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 62.77636666666666
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 37.59344980676328,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 0,
        "score": 41.76585666666666
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 37.59344980676328
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 95.78782850721109,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 98.03964210526316
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 95.78782850721109
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.75459740234666,
      86.4868177301127
     ],
     "I": [
      50.13470277647599,
      60.016870093437255
     ],
     "capability": [
      65.77545984215375,
      72.50826992991473
     ],
     "composite": null,
     "seed": 32
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.65640909439074,
     "capability": 70.24682642468954,
     "intelligence": 55.83724375498836
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7653333333333333,
      "calibration": 78.76261049180326,
      "cc": 69.68943148548576,
      "cc_by_type": {
       "choice": 69.68943148548576
      },
      "cc_tiered": 70.74726414460008,
      "cc_tiered_open": 73.67205398217818,
      "cc_tiered_sealed": 67.82247430702198,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.68943148548576
     },
     "noul": {
      "accuracy": 0.7093333333333334,
      "calibration": 88.76168525469171,
      "cc": 41.866666666666674,
      "cc_by_type": {
       "noul": 41.866666666666674
      },
      "cc_tiered": 44.265706710066624,
      "cc_tiered_open": 51.134575569358184,
      "cc_tiered_sealed": 37.396837850775064,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 41.866666666666674
     },
     "score": {
      "accuracy": 0.632,
      "calibration": 86.44493153667727,
      "cc": 50.03769070397117,
      "cc_by_type": {
       "score": 50.03769070397117
      },
      "cc_tiered": 52.49876041029837,
      "cc_tiered_open": 58.45286032699879,
      "cc_tiered_sealed": 46.544660493597945,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.03769070397117
     }
    },
    "raw_on_A": {
     "calibration": 86.62408266363812,
     "capability": 72.28243588218652,
     "intelligence": 57.94078910073491
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.58122541832599,
    "Emphasis on Accuracy 60:20:20": 62.84396292268815,
    "Emphasis on Cost 20:20:60": 68.37650147950015,
    "Emphasis on Speed 20:60:20": 75.72077375529413,
    "Intelligence only": 55.83724375498835
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Eval Engine / Chromia",
   "axes": {
    "calibration": 80.25524974800695,
    "cost": 80.41969749290011,
    "intelligence": 37.93489123230218,
    "speed": 90.61339377257747
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11555442846600955,
      "ece_score": 76.88911430679809,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 76.88911430679809,
      "score_before_time_limit": 76.88911430679809,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.27762803234501343,
      "brier": 0.14312850498011134,
      "ece": 0.09433112828523382,
      "n": 371,
      "score": 81.13377434295323,
      "score_before_time_limit": 81.13377434295323,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.907859078590786,
      "interval90_width": 0.43202348690153575,
      "n": 369,
      "nrps": 0.11270639836290829,
      "score": 82.74286059426956,
      "score_before_time_limit": 82.74286059426956,
      "time_limit_failed_share": 0,
      "top_ece": 0.11621819487585025
     }
    },
    "score": 80.25524974800695
   },
   "capability": 59.09507049015456,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 742,801 input + 0 output tokens over 1,624 decisions = 457.389778325 tokens/decision. GPU-time cost USD 0.00449445203768197/1,000; old list-price cost USD 0.0137216933497537/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00449445203768197/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004494452037681966
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decision-4B (Eval Engine / Chromia)",
   "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P88 on one evaluator-owned Lium H100. Current staged log verifies evalengine/decision-4b-gguf@08859d1d8307f7b7ea452549e303b7805c35e350; stored GGUF SHA256 f2519bcdc945291974c81a37ff92417b22df27154c9df2564420d08576b907ce. Serial offline loopback-only inference, standard selfhosted x2+0.15 s latency adjustment. Carried published estimate retained.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 39.60182572574049,
    "I_sealed": 36.267956738863866,
    "base": 37.93489123230218,
    "excess": 0.76048134443838,
    "gap": 3.333868986876624,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 65.98784234179594,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 50.92796476879523,
       "judge": 72.31377719182596,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 23.933747412008287,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 21.739130434782616,
       "judge": 0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 28.883887423417242,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.4752733983845,
       "hard": 37.0901498391321,
       "judge": 25.649863025499585,
       "standard": 7.526706201380384
      }
     },
     "sealed|choice": {
      "cc": 58.09043969346601,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 72.06414899120539,
       "hard": 47.05882352941176,
       "judge": 60.89385474860336,
       "standard": 68.96169478999877
      }
     },
     "sealed|noul": {
      "cc": 19.731441484882723,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": -12.621359223300976,
       "judge": 28.947368421052634,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 30.981989038242872,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 65.61623338387761,
       "hard": 20.121162500023082,
       "judge": 32.27985452492822,
       "standard": 33.439721711837066
      }
     }
    }
   },
   "jevbench_score": 36.9629278949271,
   "key": "evalengine-decision-4b",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "evalengine/decision-4b-gguf@08859d1d8307f7b7ea452549e303b7805c35e350",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8843283582089553,
    "decisive_rate": 0.7223719676549866,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 80.25524974800695,
     "cost": 65.87776871337948,
     "intelligence": 37.93489123230218,
     "speed": 91.06043780349009
    },
    "board_release": "v1.6.43",
    "capability": 59.09507049015456,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 63,
    "capability_rank": 88,
    "composite_ci95": {
     "A": [
      21.75506402712618,
      47.346177294331085
     ],
     "B": [
      18.702949452781887,
      42.97523990261764
     ],
     "C": [
      15.107683352170962,
      32.879289787729924
     ]
    },
    "jevbench_score": 35.43243109909669,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 61,
    "rank": 91,
    "ranks": {
     "A": 91,
     "B": 92,
     "C": 92,
     "capability": 137
    },
    "scores": {
     "A": 35.43243109909669,
     "B": 31.508666798854446,
     "C": 24.60585492992826
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24496516967192292,
     "p50_s_raw": 0.04748258483596146,
     "p95_s_adjusted": 0.3197804291266948,
     "p95_s_raw": 0.08489021456334739
    },
    "views": {
     "Balanced 33:33:33": 32.87874158865029,
     "Emphasis on Accuracy 60:20:20": 27.346985238704878,
     "Emphasis on Cost 20:20:60": 34.72560202914835,
     "Emphasis on Speed 20:60:20": 38.639773156510174,
     "Intelligence only": 21.836172722245216
    }
   },
   "provenance": {
    "carried_metadata_from": "Current inventory + r20 source metadata; unresolved runtime join",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 154,
   "ranked": true,
   "ranks": {
    "A": 154,
    "B": 155,
    "C": 155,
    "capability": 170
   },
   "repo": "https://huggingface.co/evalengine/decision-4b-gguf",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 36.9629278949271,
    "B": 32.46498025340702,
    "C": 25.66869992703271
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26147820902988317,
    "p50_s_raw": 0.05573910451494157,
    "p95_s_adjusted": 0.33206654312089084,
    "p95_s_raw": 0.09103327156044543
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.03864735316435,
      82.85641931336254
     ],
     "I": [
      31.11663059494919,
      42.88120054683658
     ],
     "capability": [
      54.031339191317976,
      62.320753924360694
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 80.25524974800695,
     "capability": 59.09507049015456,
     "intelligence": 37.93489123230218
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6866666666666666,
      "calibration": 76.88911430679809,
      "cc": 59.92914183947025,
      "cc_by_type": {
       "choice": 59.92914183947025
      },
      "cc_tiered": 62.039141017630975,
      "cc_tiered_open": 65.98784234179594,
      "cc_tiered_sealed": 58.09043969346601,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.92914183947025
     },
     "noul": {
      "accuracy": 0.632,
      "calibration": 81.13377434295323,
      "cc": 26.400000000000002,
      "cc_by_type": {
       "noul": 26.400000000000002
      },
      "cc_tiered": 21.832594448445505,
      "cc_tiered_open": 23.933747412008287,
      "cc_tiered_sealed": 19.731441484882723,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 26.400000000000002
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 82.74286059426956,
      "cc": 32.24707656205765,
      "cc_by_type": {
       "score": 32.24707656205765
      },
      "cc_tiered": 29.932938230830057,
      "cc_tiered_open": 28.883887423417242,
      "cc_tiered_sealed": 30.981989038242872,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 32.24707656205765
     }
    },
    "raw_on_A": {
     "calibration": 78.19048224326764,
     "capability": null,
     "intelligence": null
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 34.65404663389847,
    "Emphasis on Accuracy 60:20:20": 28.064484775676526,
    "Emphasis on Cost 20:20:60": 38.528350802725136,
    "Emphasis on Speed 20:60:20": 40.02747561698624,
    "Intelligence only": 21.836172722245216
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Dongmin Kim / @carrtesy (tommy.dm.kim@gmail.com) - unofficial personal project",
   "axes": {
    "calibration": 80.52861474872404,
    "cost": 57.919479713067595,
    "intelligence": 14.831288392065307,
    "speed": 93.81713750966469
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1631615191239309,
      "ece_score": 67.36769617521382,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 67.36769617521382,
      "score_before_time_limit": 67.36769617521382,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.5983827493261455,
      "brier": 0.1968188345782441,
      "ece": 0.06151585075654434,
      "n": 371,
      "score": 87.69682984869114,
      "score_before_time_limit": 87.69682984869114,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.948509485094851,
      "interval90_width": 0.7529358626919599,
      "n": 369,
      "nrps": 0.15012627140838788,
      "score": 86.52131822226714,
      "score_before_time_limit": 86.52131822226714,
      "time_limit_failed_share": 0,
      "top_ece": 0.05972368207313468
     }
    },
    "score": 80.52861474872404
   },
   "capability": 47.67995157039467,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     1.695484672747176,
     5.268678060115385
    ],
    "B": [
     1.2625009050434257,
     4.159710667472831
    ],
    "C": [
     1.1774199116299833,
     3.6588042084134624
    ]
   },
   "cost": {
    "basis": "ESTIMATE (size-class reference; self-hosted open weights, no bookable listing for this base). THE RATE IS RUN 48'S, CARRIED UNCHANGED FOR THE SAME BASE: run 48 priced exaone-4.0-1.2b-jev (v0.2, identical base) at USD 0.02 per 1M input and 0 output, as the HIGHER of the two bracketing published rates, chosen in that direction because the rate was fixed after the measurement existed. Keeping it makes v0.3 directly comparable with v0.2 and takes no new decision. The frozen 25 Sep snapshot still contains NO listing for LGAI-EXAONE/EXAONE-4.0-1.2B (pricing_v15.reference() raises: 'base model lacks an explicit market-listing decision'), and still brackets 1.2B between deepinfra:Qwen/Qwen3.5-0.8B at (0.01, 0.05) and deepinfra:Qwen/Qwen3.5-2B at (0.02, 0.10). Estimated, not charged. PRICING REFERENCE NOTE: LGAI-EXAONE/EXAONE-4.0-1.2B must be adopted into the frozen UNLISTED_BASE_MODELS table before either EXAONE row may be published. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone and any other decision is reversible by arithmetic from the raw.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.02488704
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025274583615436694
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "EXAONE-4.0-1.2B-JEV v0.3 (unofficial fine-tune of LGAI-EXAONE/EXAONE-4.0-1.2B, native option-label softmax, two reads averaged, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), 1,477 of 1,500 answered. NEW REVISION of the row run 48 measured: weights carrtesy/EXAONE-4.0-1.2B-JEV at TAG v0.3, server github.com/carrtesy/EXAONE-JEV at tag v0.3 (commit recorded in receipts/POD-R58.json), both the artefacts the author's own submit-form entry names. EXAONE AI Model License 1.2 - NC, NON-COMMERCIAL, and an unofficial personal project - the author states it is 'not an LG AI Research release'; both belong on the row. NO MAPPING DECISION OF OURS: their serve/systemone_server.py speaks the TypeSafe Jev POST /v1/systemone contract, so the UNCHANGED jevbench.adapters.typesafe emits exactly what it parses, and all eight checks of the unamended smoke gate passed on synthetic requests before any sealed item was sent. Read count (permute 2: options in the given and the reversed order, distributions averaged) and the per-type temperatures come from the weights' own jev_config.json, which the author's doc states needs no flags - their shipped default, not a setting of ours. STACK: the author's own docs/REPRODUCE.md install line in an isolated venv, vllm==0.30.0 letting it resolve its own stack (torch 2.13.0+cu130, transformers 5.19.0), served with their own published command verbatim (`python serve/systemone_server.py --model weights --port 8000 --gpu-util 0.9`) and VLLM_USE_FLASHINFER_SAMPLER=0 as their doc sets it. One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW REFUSES THEM, at the limit the author DECLARES: their docs/REPRODUCE.md section 3 states a 65,536-token prompt limit answering 400, and all 23 came back as 'HTTP 400: prompt of 84996 tokens is longer than the maximum context length 65536'. The 23 invalid are therefore the system's own disclosed refusal, counted wrong, and no other item failed. Serial, one decision per request, no retry, no batching, offline, credential-free. add-requests run 58, benchmarkheaven.com/submit ref CA53D793 (bh-submit 5631d258-1eaa-4e2d-a658-f6417b85b351); previously GitHub issue #176",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090 32 GB",
   "intelligence": {
    "I_open": 16.786521241837672,
    "I_sealed": 12.87605554229294,
    "base": 14.831288392065307,
    "excess": 1.337078057106488,
    "gap": 3.9104656995447318,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 39.40654352431394,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 49.40540540540541,
       "hard": 20.729789241899976,
       "judge": 40.672379696769944,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.28571428571429,
       "hard": -47.82608695652174,
       "judge": -40,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 10.953020201199083,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.5784737243812,
       "hard": -2.3251786244617323,
       "judge": 27.73524201685903,
       "standard": -0.4766416325602707
      }
     },
     "sealed|choice": {
      "cc": 28.792593942604675,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 45.458576601877176,
       "hard": 18.04298642533935,
       "judge": 32.96089385474861,
       "standard": 35.70636777928318
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 15.999999999999993,
       "hard": -63.106796116504846,
       "judge": -42.10526315789473,
       "standard": -9.859154929577462
      }
     },
     "sealed|score": {
      "cc": 9.835572684274151,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 36.004469283378306,
       "hard": -2.325764560565169,
       "judge": 14.49323221885701,
       "standard": 14.08730957252643
      }
     }
    }
   },
   "jevbench_score": 3.265807063300941,
   "key": "exaone-4.0-1.2b-jev-v0.3",
   "last_measured_on": "2026-10-07",
   "licence": "EXAONE AI Model License 1.2 (non-commercial)",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "carrtesy/EXAONE-4.0-1.2B-JEV@v0.3",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8456375838926175,
    "decisive_rate": 0.40161725067385445,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 80.52861474872404,
     "cost": 57.919479713067595,
     "intelligence": 14.831288392065307,
     "speed": 94.09940251974388
    },
    "board_release": "v1.6.43",
    "capability": 47.67995157039467,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 95,
    "capability_rank": 122,
    "composite_ci95": {
     "A": [
      1.6959116121891469,
      5.270428159114469
     ],
     "B": [
      1.262690264794709,
      4.160583281665687
     ],
     "C": [
      1.1777163973535743,
      3.660019554940604
     ]
    },
    "jevbench_score": 3.266776278413127,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 151,
    "rank": 191,
    "ranks": {
     "A": 191,
     "B": 194,
     "C": 192,
     "capability": 181
    },
    "scores": {
     "A": 3.266776278413127,
     "B": 2.5116077374172012,
     "C": 2.2685946377868937
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19053296000347472,
     "p50_s_raw": 0.020266480001737364,
     "p95_s_adjusted": 0.20421593803039284,
     "p95_s_raw": 0.02710796901519643
    },
    "views": {
     "Balanced 33:33:33": 2.7692786828242713,
     "Emphasis on Accuracy 60:20:20": 1.9113656734726492,
     "Emphasis on Cost 20:20:60": 3.388067484365469,
     "Emphasis on Speed 20:60:20": 3.7739415394848543,
     "Intelligence only": 1.304958289921029
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 233,
   "ranked": true,
   "ranks": {
    "A": 233,
    "B": 234,
    "C": 234,
    "capability": 219
   },
   "repo": "https://huggingface.co/carrtesy/EXAONE-4.0-1.2B-JEV",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.265807063300941,
    "B": 2.5111493585324483,
    "C": 2.267921571736765
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19166760700172744,
    "p50_s_raw": 0.020833803500863723,
    "p95_s_adjusted": 0.21663944199390245,
    "p95_s_raw": 0.03331972099695122
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "LGAI-EXAONE/EXAONE-4.0-1.2B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      74.05366116439995,
      82.35803770559488
     ],
     "I": [
      11.603293028025588,
      17.810851432602874
     ],
     "capability": [
      43.59835541237688,
      49.150688153062475
     ],
     "composite": {
      "A": [
       1.695484672747176,
       5.268678060115385
      ],
      "B": [
       1.2625009050434257,
       4.159710667472831
      ],
      "C": [
       1.1774199116299833,
       3.6588042084134624
      ]
     },
     "seed": 53
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 80.52861474872404,
     "capability": 47.67995157039467,
     "intelligence": 14.831288392065307
    },
    "per_type": {
     "choice": {
      "accuracy": 0.45866666666666667,
      "calibration": 67.36769617521382,
      "cc": 30.771198241808182,
      "cc_by_type": {
       "choice": 30.771198241808182
      },
      "cc_tiered": 34.09956873345931,
      "cc_tiered_open": 39.40654352431394,
      "cc_tiered_sealed": 28.792593942604675,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 30.771198241808182
     },
     "noul": {
      "accuracy": 0.336,
      "calibration": 87.69682984869114,
      "cc": -32.8,
      "cc_by_type": {
       "noul": -32.8
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.3893333333333333,
      "calibration": 86.52131822226714,
      "cc": 10.441024816632405,
      "cc_by_type": {
       "score": 10.441024816632405
      },
      "cc_tiered": 10.394296442736618,
      "cc_tiered_open": 10.953020201199083,
      "cc_tiered_sealed": 9.835572684274151,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 10.441024816632405
     }
    },
    "raw_on_A": {
     "calibration": 76.1279082946144,
     "capability": 45.86026412407656,
     "intelligence": 15.592619953538719
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.7683500657470317,
    "Emphasis on Accuracy 60:20:20": 1.9111001961152008,
    "Emphasis on Cost 20:20:60": 3.387233424113146,
    "Emphasis on Speed 20:60:20": 3.7708387334363804,
    "Intelligence only": 1.304958289921029
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Fastino",
   "axes": {
    "calibration": 87.95,
    "cost": 35.86,
    "intelligence": 66.1,
    "speed": 77.23
   },
   "calibration": {
    "score": 87.95
   },
   "capability": 77.02,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     29.46,
     31.56
    ],
    "B": [
     29.6,
     32.79
    ],
    "C": [
     27.79,
     31.56
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.15/M input, 0/M output (spec fastino-glide.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.13735482233502538
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Fastino GLiDE",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 64.15,
    "I_sealed": 62.91,
    "base": 66.1,
    "per_type_split": {
     "open|choice": {
      "cc": 92.7738,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100,
       "hard": 84.9009,
       "judge": 96.0448,
       "standard": 100
      }
     },
     "open|noul": {
      "cc": 34.3685,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 47.8261,
       "judge": 0,
       "standard": 33.3333
      }
     },
     "open|score": {
      "cc": 65.3016,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 78.1513,
       "hard": 64.5629,
       "judge": 75.2145,
       "standard": 45.485
      }
     },
     "sealed|choice": {
      "cc": 93.9895,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 100,
       "hard": 88.3658,
       "judge": 95.4774,
       "standard": 100
      }
     },
     "sealed|noul": {
      "cc": 36.4305,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 52.9412,
       "hard": 9.0909,
       "judge": 50,
       "standard": 62.5
      }
     },
     "sealed|score": {
      "cc": 58.3168,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 87.3406,
       "hard": 37.016,
       "judge": 63.7483,
       "standard": 78.2591
      }
     }
    }
   },
   "jevbench_score": 30.57,
   "key": "fastino-glide",
   "last_measured_on": "2026-10-06",
   "licence": "hosted API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 32,
    "axes": {
     "calibration": 87.95,
     "cost": 35.86,
     "intelligence": 66.1,
     "speed": 77.48
    },
    "board_release": "v1.6.43",
    "capability": 77.02,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      29.48,
      31.58
     ],
     "B": [
      29.61,
      32.81
     ],
     "C": [
      27.81,
      31.58
     ]
    },
    "jevbench_score": 30.59,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 99,
    "ranks": {
     "A": 99,
     "B": 94,
     "C": 76,
     "capability": 39
    },
    "scores": {
     "A": 30.59,
     "B": 31.21,
     "C": 30.59
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.743,
     "p50_s_raw": 0.743,
     "p95_s_adjusted": 2.405,
     "p95_s_raw": 2.405
    },
    "views": null
   },
   "rank": 164,
   "ranked": true,
   "ranks": {
    "A": 164,
    "B": 157,
    "C": 141,
    "capability": 52
   },
   "repo": null,
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 30.57,
    "B": 31.2,
    "C": 30.57
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.852,
    "p50_s_raw": 0.852,
    "p95_s_adjusted": 2.218,
    "p95_s_raw": 2.218
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "a5736558e4af448fa482a03cbcbd4b119fc82c7695931ca18b671b86dfe1e724"
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Fastino (Hugging Face)",
   "axes": {
    "calibration": 93.46414212015411,
    "cost": 58.10251223805742,
    "intelligence": 76.38100890445796,
    "speed": 85.49903308442846
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03759375786907435,
      "ece_score": 92.48124842618512,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 92.48124842618512,
      "score_before_time_limit": 92.48124842618512,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.13746630727762799,
      "brier": 0.05329684013928134,
      "ece": 0.04180203505001547,
      "n": 371,
      "score": 91.63959298999691,
      "score_before_time_limit": 91.63959298999691,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9892183288409704,
      "interval90_width": 0.21406109613656782,
      "n": 371,
      "nrps": 0.03790344886809453,
      "score": 96.2715849442803,
      "score_before_time_limit": 96.2715849442803,
      "time_limit_failed_share": 0,
      "top_ece": 0.018332426123149732
     }
    },
    "score": 93.46414212015411
   },
   "capability": 84.92257551230603,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11861.3382645 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=trunk-equivalent). The row's own scored cost basis: 724,641 input + 0 output tokens over 1,482 decisions = 488.961538462 tokens/decision. GPU-time cost USD 0.0249220019347192/1,000; old list-price cost USD 0.205363846153846/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0249220019347192/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1482,
     "rule": "common",
     "usd_per_1000_all_items": 0.20330609218436874
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.024922001934719162
   },
   "custody_exclusion": {
    "derived_gold_sha256": "9f1f8200d0a1b4384f43ffc0a91f3f36e4790fc3dbcef7486bf233b0142baa47",
    "exclusion_list_sha256": "31b47af58dce134a70e5cbf60aad40db0000dd398f26649070fb7c9dfd1deeb3",
    "items": 3,
    "rule": "an item a provider has already seen is never scored for that provider; the 3 items were never sent and are not in this row's gold",
    "scored_items": 1497
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1497,
    "treatment": "share"
   },
   "display": "Fastino GLiDE 1.2 (open weights, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.574/h including disk. The authors' server (127.0.0.1:8000) in ONE --network none --read-only container; driver requests serial over loopback. Read-only code, model and input, empty credential variables, HF offline flags; cut proof inside the container before any sealed input (only loopback, no default route, DNS and TCP fail). Measured 23 Oct 2026 (VM reserved 08:58:49Z, scored loop started 09:05:43Z, teardown confirmed 09:10:37Z); this row measurement 277 s, scored loop 227.5 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 6ab4c6deab3cffc6e9fb7f176f52250b20d6ee936dabc9711a9858370447432a. Node-shred receipt verified; exact-id VM teardown confirms the provider instance absent and the guard closed. GPU rental is not the benchmark's estimated model tariff. Earlier attempts of this same row made no sealed contact: attempt 0 failed before any sealed byte on a runner bug (a JSON false in Python source in the import check; fixed and reviewed), and two Verda create calls answered HTTP 503 with no instance; the one exposure use (on file) was consumed by this run.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 79.71533523123628,
    "I_sealed": 73.04668257767963,
    "base": 76.38100890445796,
    "excess": -3.1708741167990624,
    "gap": 6.668652653556649,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 90.51495356571994,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 86.78829820698331,
       "judge": 96.044825313118,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 75.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 70,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 72.68281196235743,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.58216426950466,
       "hard": 72.46893187870765,
       "judge": 82.88762650645698,
       "standard": 58.35367415993408
      }
     },
     "sealed|choice": {
      "cc": 87.0566039972978,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 82.30600208164681,
       "judge": 86.77766895200784,
       "standard": 91.45749841805527
      }
     },
     "sealed|noul": {
      "cc": 63.1141990537663,
      "n": {
       "easy": 58,
       "hard": 109,
       "judge": 64,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 50.458715596330286,
       "judge": 68.75,
       "standard": 70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 68.9692446819748,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 66,
       "standard": 75
      },
      "tiers": {
       "easy": 86.472785214709,
       "hard": 64.98331797139856,
       "judge": 69.06085764182093,
       "standard": 68.0519083969909
      }
     }
    }
   },
   "jevbench_score": 75.9034188361358,
   "key": "fastino-glide-1.2-open",
   "last_measured_on": "2026-10-23",
   "licence": "Fastino Community License for Fastino's modifications and weights (LICENSE and NOTICE at the pin; benchmarking is outside 'Production'; attribution line 'Built with Fastino GLiDE' for Commercial Use), Qwen3.8-27B parts Apache-2.0 (licenses/Qwen3.8-27B-LICENSE); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "fastino/GLiDE@a7e45896be7c96ba3354891da790c5ec99c30b6c",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.975,
    "decisive": 320,
    "decisive_rate": 0.862533692722372,
    "n": 373,
    "supported": true,
    "valid": 371
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 10,
   "ranked": true,
   "ranks": {
    "A": 10,
    "B": 7,
    "C": 9,
    "capability": 9
   },
   "repo": "https://huggingface.co/fastino/GLiDE",
   "scores": {
    "A": 75.9034188361358,
    "B": 75.99845845261994,
    "C": 75.9034188361358
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1497,
    "p50_s_adjusted": 0.4293506360000038,
    "p50_s_raw": 0.13967531800000188,
    "p95_s_adjusted": 0.6565753607999796,
    "p95_s_raw": 0.2532876803999898
   },
   "status": {
    "answered_ok": 1482,
    "missing": 0,
    "rows": 1497,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run as shipped: fastino/GLiDE at a7e45896be7c96ba3354891da790c5ec99c30b6c (Hub main at the pin; the five glide/*.py files are served unmodified from the hash-verified snapshot folder, sha256 __init__.py 7e807d7d94895d436c43c2b2add0521608e208c5dde7e0016549c5b8ee5d7192, images.py 4858f30b92819abe463d845886f3240145c532fd0dfc725f5a08586e7591c616, model.py dd771ffe6c354e4a46dd1e4d8573be4faf0e8ca416010757dd2de4e5724172b2, prompt.py e47149ac980fe12c3cb24ea0ee63822d5bcde71f32d4a5cc9f37499853948338, server.py baa81296446828455b0ca1b43e58f3662b7a06ab684e5ef02d0e4fa77574030a; re-hashed inside the sealed container). Command: env PYTHONPATH=<snapshot> PYTHONDONTWRITEBYTECODE=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 USE_HUB_KERNELS=NO GLIDE_MODEL_NAME=glide-v1.2 GLIDE_MAX_BATCH=1 GLIDE_MAX_LEN=40000 GLIDE_API_KEY= python3 -m glide.server --model <snapshot> --host 127.0.0.1 --port 8000. bf16, PyTorch SDPA attention, one request per forward pass, temperature 1.0 from decision_config.json (sha256 df40c569d6248a8562177ea8377d47c6fbe558521f60dd14e4e8a78d04e7ce10), no kernel packages (flash-linear-attention, causal-conv1d and Hub kernels absent; checked in the container). Image: vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 used only as a CUDA 13.0 / PyTorch base (vLLM is not used); derived image sha256:f8d7c68d21d235430e49996d047c1175da52a16841fddca82fbe21fbd662f9ce, built offline from four hash-pinned wheels (transformers 5.17.0, huggingface-hub 1.33.0, fastapi 0.136.3, uvicorn 0.54.0). Python 3.12.3, torch 2.13.0+cu130, tokenizers 0.23.2, numpy 2.2.6. DEVIATIONS from the author's requirements.txt, asserted before the run: torch 2.13.0 (author 2.14.0), numpy 2.2.6 (2.5.3), fastapi 0.136.3 (0.142.2). Code gate: cleared by the benchmark owner on the disclosed patterns. DISCLOSED PATTERNS: one dormant Hub download line, snapshot_download at glide/model.py:216-218, reached only if --model is not a directory (we pass the local snapshot folder); dormant Hub-kernel loaders in the transformers Qwen3.5 code, inert with USE_HUB_KERNELS=NO and no kernels package. No trust_remote_code, subprocess, eval/exec, pickle, telemetry or request-body logging in the author code. The source and runner were reviewed before the run, findings were fixed and the final changes rechecked. Weights: Fastino Community License, public, ungated, 27 files, 52,193,616,460 bytes, every file hash-checked on the node (VERIFY OK 27 files); manifest sha256 5e9fdbcf53ffd2ae843d025e62745a145672283a0bada42bc3e90779d277d0de. The Hub main moved to e827fc56 on 23 Oct 2026 07:12Z ('Update README.md'; only README.md changed, one line making a LICENSE link absolute); the measurement stays at the pinned commit a7e45896. FIRST MEASUREMENT OF THE OPEN WEIGHTS, A NEW ROW (key fastino-glide-1.2-open; evaluator exposure and device ruling). NOT TO BE CONFUSED WITH THE TWO HOSTED ROWS: fastino-glide (hosted API, first measured 6 Oct 2026 in v1.6.1 on 600 items) and fastino-glide-api (hosted API, 22 Oct 2026 on 1,497 items; Fastino's API reports its model as glide_v2). This row is the open-weights checkpoint fastino/GLiDE at a7e45896 (version 1.2.0, engine name glide-v1.2, first uploaded 19 Oct 2026), served by us on our own rented GPU through the authors' own server; as of 22 Oct 2026 nothing published says that either hosted row served this checkpoint, and the hosted rows differ in prompt limit (65,536 vs 40,000 tokens), tariff and engine name. SCORED ON 1,497 OF THE DRAW'S 1,500 ITEMS: all 1,500 items were SENT unchanged (input hash unchanged), but 3 items Fastino had already seen through its hosted GLiNER-2.5-Decide on 8 Oct 2026 are never scored for this row (exposure rule: an item a provider has seen is never scored for that provider). The row is scored with a second invocation of the unmodified frozen scorer on a derived gold that lacks exactly those 3 ids (exclusion file sha256 31b47af58dce134a70e5cbf60aad40db0000dd398f26649070fb7c9dfd1deeb3; the derived gold's sha256 is recorded in the release proof); the other rows of this board are scored on all 1,500 items. NOT ANSWERED, counted wrong: 15 items exceed the author's 40,000-token context limit (refused above 39,992 prompt tokens; HTTP 422 'prompt exceeds the maximum context length'), exactly the 15 standard very long items (76,339-80,497 tokens) at positions 65, 169, 474, 482, 515, 573, 580, 729, 833, 899, 951, 1057, 1116, 1198, 1418, as pre-registered by id and position; never truncated. Answered: 1,485 of the 1,500 items sent with HTTP 200 (status counts 200: 1,485, 422: 15); of the 1,497 scored items, 1,482 were answered and the same 15 refused. AUTHOR'S OWN SERVER on our rented hardware, bf16, ONE REQUEST PER FORWARD PASS (GLIDE_MAX_BATCH=1). NATIVE OUTPUTS: choice argmax and probabilities, Noul P(true), score level and expectation, all from the 255-way readout softmax at temperature 1.0; no rotation or calibration file, nothing generated. OPTIONAL KERNELS OFF, FIDELITY RISK: the author's validation and all numbers in its model card used flash-linear-attention; this run used the PyTorch fallback, which the author says 'was not part of the validation', so small bf16 differences in probabilities against the author's own runs are possible and were not measured. The reason was the 10-second rule: a cold Triton cache adds 8-35 s per new shape according to the model card and the sealed container starts empty. Three package versions differ from the author's pins (see code_revision). AUTHOR EXPOSURE: the model card at the pinned revision makes no statement about training data, JevBench or decontamination; no overlap or contamination audit was run by us, and no author statement was requested or received. LICENCE NOTES: the Fastino Community License excludes evaluation, testing, benchmarking and research from 'Production' (section 6(b)); its section 4 asks for the line 'Built with Fastino GLiDE' on a related website for any Commercial Use, and the model card's introduction says 'Internal use only.' Built with Fastino GLiDE. This result does not establish that the hosted fastino-glide-api row uses the same weights. The model was under Apache-2.0 for about 50 minutes on 22 Oct 2026 before the licence changed; weights, code and configuration are byte-identical in all five revisions up to the pin (only README.md and licence files changed). 10 s RULE: 0 decisions over 10 s, maximum measured decision latency 0.485 s over the 1,500 requests, so the 10-second limit of the v1.7 share view costs this row nothing (the frozen v1.6 and v1.7-off views apply none). The longest answered prompt was 1,839 tokens (pre-registered offline count). For context, the pre-run synthetic probes (not benchmark items) took 5.3 s at 16,000 tokens and 17.06 s at the 39,992-token limit. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. CONTEXT: author-imposed limit 40,000 tokens (GLIDE_MAX_LEN, default); no truncation. The vendor's reported figures and the Decision Index figure for this checkpoint are not independently verified by us.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 93.28683402489601,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 88.19383540802139
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 93.28683402489601
      },
      "calibration_gen": {
       "accuracy": 0.7920792079207921,
       "cc": 61.10934993664238,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 43.99999999999999,
        "score": 64.77873909942872
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.10934993664238
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 88.20348809055965,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 75.26990419941157
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 88.20348809055965
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.29867459427453,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 98.89602378282356
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.29867459427453
      },
      "jaggedness": {
       "accuracy": 0.7944444444444444,
       "cc": 58.71369167107085,
       "cc_by_type": {
        "choice": 79.78825794032724,
        "noul": 18.918918918918926,
        "score": 77.43389815396637
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 58.71369167107085
      },
      "llm_router": {
       "accuracy": 0.9552715654952076,
       "cc": 88.30660149539369,
       "cc_by_type": {
        "choice": 97.62261160517741,
        "noul": 100,
        "score": 67.29719288100368
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 88.30660149539369
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.75,
       "cc": 52.56750812051602,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 33.33333333333333,
        "score": 49.11145906945185
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 6,
        "score": 7
       },
       "score": 52.56750812051602
      },
      "multilingual": {
       "accuracy": 0.8181818181818182,
       "cc": 69.36083093459378,
       "cc_by_type": {
        "choice": 84.3495272253016,
        "noul": 55.55555555555556,
        "score": 68.17741002292419
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.36083093459378
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 74.46799128621208,
       "cc_by_type": {
        "choice": 100,
        "noul": 73.33333333333334,
        "score": 50.07064052530291
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.46799128621208
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 82.54058281009027,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 80,
        "score": 79.63376044228282
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 82.54058281009027
      },
      "usecase": {
       "accuracy": 0.8545454545454545,
       "cc": 73.9863191228161,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 58.33333333333333,
        "score": 73.42954560374241
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 73.9863191228161
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 49.037941029788264,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50,
        "score": 16.16144213698383
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 49.037941029788264
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 55.383081739866704,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 32.81591188626678
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 55.383081739866704
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 90.75212945562821,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 88.18559190670764
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 90.75212945562821
      },
      "el": {
       "accuracy": 0.9333333333333333,
       "cc": 94.3823560381911,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100,
        "score": 99.90684465088616
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 94.3823560381911
      },
      "en": {
       "accuracy": 0.8748913987836664,
       "cc": 76.71598640342684,
       "cc_by_type": {
        "choice": 88.79681722214764,
        "noul": 69.06474820143885,
        "score": 72.28639378669403
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 76.71598640342684
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 73.96984447427957,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 31.69746164469842
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.96984447427957
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.72662140938952,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.84653089483527
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.72662140938952
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 92.11359084699383,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 97.51724312921681
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 92.11359084699383
      },
      "hi": {
       "accuracy": 0.8888888888888888,
       "cc": 70.43840994806452,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 66.66666666666667,
        "score": 44.64856317752691
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 4
       },
       "score": 70.43840994806452
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 66.79664710844526,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 14.41955940795463
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 66.79664710844526
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 52.487729250622465,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 37.4631877518674
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 52.487729250622465
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 18.81075830924279,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 56.43227492772836
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 18.81075830924279
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.9712888346815,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.91386650404452
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.9712888346815
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 1.9262230233236461,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": -34.91397162266976
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 1.9262230233236461
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 78.59736445947553,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80,
        "score": 71.02696430521456
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 78.59736445947553
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 78.12012789523055,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 66.18124402747125
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 78.12012789523055
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 60.220633886041526,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 46.19381655174158
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 60.220633886041526
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 86.05162469672574,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 79.8940045249598
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 86.05162469672574
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 83.92582991138106,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 84.03555425027218
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 83.92582991138106
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 65.47290998186114,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 63.08539661225008
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 65.47290998186114
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.908,
       "cc": 88.11687938919613,
       "cc_by_type": {
        "choice": 88.11687938919613
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 88.11687938919613
      },
      "noul": {
       "accuracy": 0.8364611260053619,
       "cc": 67.29222520107238,
       "cc_by_type": {
        "noul": 67.29222520107238
       },
       "low_n": false,
       "n": 373,
       "n_by_type": {
        "noul": 373
       },
       "score": 67.29222520107238
      },
      "score": {
       "accuracy": 0.8021390374331551,
       "cc": 70.23031955117594,
       "cc_by_type": {
        "score": 70.23031955117594
       },
       "low_n": false,
       "n": 374,
       "n_by_type": {
        "score": 374
       },
       "score": 70.23031955117594
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 28.69392249485216,
       "cc_by_type": {
        "choice": 68,
        "noul": -25,
        "score": 43.08176748455649
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 28.69392249485216
      },
      "code_linting": {
       "accuracy": 1,
       "cc": 93.48129249546712,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 80.44387748640132
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 93.48129249546712
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 63.834484500614856,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 24.0237787050966
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 63.834484500614856
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": -0.8645333531519933,
       "cc_by_type": {
        "choice": 53.21637426900585,
        "noul": 33.33333333333333,
        "score": -89.14330766179515
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.875,
       "cc": 68.38791810434252,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 86.32317460288262
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 68.38791810434252
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 56.21750624743703,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": 47.89780176117902
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 56.21750624743703
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 51.137965239512276,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -14.28571428571429,
        "score": 67.69961000425111
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 51.137965239512276
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 52.67279304179414,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 58.01837912538241
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 52.67279304179414
      },
      "insurance_claims": {
       "accuracy": 0.6428571428571429,
       "cc": 62.805898885724105,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 50,
        "score": 59.25102999050564
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 4
       },
       "score": 62.805898885724105
      },
      "knowledge_graphs": {
       "accuracy": 0.9545454545454546,
       "cc": 90.58078803635563,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 71.74236410906687
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 90.58078803635563
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 67.07503322126443,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 57.297451084981944
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.07503322126443
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 34.89585122275884,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 68.12841388333031
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 34.89585122275884
      },
      "llm_guardrails": {
       "accuracy": 0.9259259259259259,
       "cc": 80.84937867696657,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 82.54813603089968
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 80.84937867696657
      },
      "model_routing": {
       "accuracy": 0.9351851851851852,
       "cc": 84.84238463865871,
       "cc_by_type": {
        "choice": 96.49350649350649,
        "noul": 100,
        "score": 58.033647422469684
       },
       "low_n": false,
       "n": 216,
       "n_by_type": {
        "choice": 101,
        "noul": 65,
        "score": 50
       },
       "score": 84.84238463865871
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 54.98952694592623,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 88.00121039631993
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 54.98952694592623
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 67.86249205295886,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 70.25414282554324
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 67.86249205295886
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 62.175293189844275,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 36.52587956953282
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 62.175293189844275
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 55.986205509765625,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 63.610790442340345
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 55.986205509765625
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.92032149163197,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.7609644748959
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.92032149163197
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      89.20818030834448,
      93.61993386336131
     ],
     "I": [
      72.30102290134639,
      79.94449290262376
     ],
     "capability": [
      81.44074129934772,
      86.23327088510355
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1197,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1497,
    "n_public": 300,
    "n_sealed_side": 1197,
    "on_S": {
     "calibration": 93.46414212015411,
     "capability": 84.92257551230603,
     "intelligence": 76.38100890445796
    },
    "per_type": {
     "choice": {
      "accuracy": 0.908,
      "calibration": 92.48124842618512,
      "cc": 88.11687938919613,
      "cc_by_type": {
       "choice": 88.11687938919613
      },
      "cc_tiered": 88.78577878150887,
      "cc_tiered_open": 90.51495356571994,
      "cc_tiered_sealed": 87.0566039972978,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 88.11687938919613
     },
     "noul": {
      "accuracy": 0.8364611260053619,
      "calibration": 91.63959298999691,
      "cc": 67.29222520107238,
      "cc_by_type": {
       "noul": 67.29222520107238
      },
      "cc_tiered": 69.5312196096989,
      "cc_tiered_open": 75.94824016563147,
      "cc_tiered_sealed": 63.1141990537663,
      "low_n": false,
      "n": 373,
      "n_by_type": {
       "noul": 373
      },
      "score": 67.29222520107238
     },
     "score": {
      "accuracy": 0.8021390374331551,
      "calibration": 96.2715849442803,
      "cc": 70.23031955117594,
      "cc_by_type": {
       "score": 70.23031955117594
      },
      "cc_tiered": 70.82602832216611,
      "cc_tiered_open": 72.68281196235743,
      "cc_tiered_sealed": 68.9692446819748,
      "low_n": false,
      "n": 374,
      "n_by_type": {
       "score": 374
      },
      "score": 70.23031955117594
     }
    },
    "raw_on_A": {
     "calibration": 89.83389322509132,
     "capability": 82.01303424195552,
     "intelligence": 74.19217525881973
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.00536193029490617,
     "n": 373
    },
    "score": {
     "invalid_rate": 0.008021390374331583,
     "n": 374
    }
   },
   "views": {
    "Balanced 33:33:33": 71.42983318657053,
    "Emphasis on Accuracy 60:20:20": 73.33122617204468,
    "Emphasis on Cost 20:20:60": 65.4268902409589,
    "Emphasis on Speed 20:60:20": 76.46272952524222,
    "Intelligence only": 76.38100890445796
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Fastino",
   "axes": {
    "calibration": 92.84353248115252,
    "cost": 59.111125613009825,
    "intelligence": 76.06842865080436,
    "speed": 87.00413908296701
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04346043152131592,
      "ece_score": 91.30791369573681,
      "mean_tvd": null,
      "n": 738,
      "n_ece_hard": 304,
      "n_tvd": 0,
      "score": 91.30791369573681,
      "score_before_time_limit": 91.30791369573681,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.13746630727762799,
      "brier": 0.0529066276825842,
      "ece": 0.03815484610642815,
      "n": 371,
      "score": 92.12139262649261,
      "score_before_time_limit": 92.36903077871436,
      "time_limit_failed_share": 0.002680965147453083
     },
     "score": {
      "interval90_coverage": 0.9892183288409704,
      "interval90_width": 0.21428571428571425,
      "n": 371,
      "nrps": 0.0377969402276481,
      "score": 95.10129112122816,
      "score_before_time_limit": 95.35625436820196,
      "time_limit_failed_share": 0.00267379679144385,
      "top_ece": 0.02753898620415633
     }
    },
    "score": 92.84353248115252
   },
   "capability": 84.45598056597845,
   "composite_ci95": {
    "A": [
     74.90033635084451,
     77.08874894746072
    ],
    "B": [
     74.3293214235386,
     77.56901714437397
    ],
    "C": [
     74.90033635084451,
     77.08874894746072
    ]
   },
   "cost": {
    "basis": "Operator's own published standard usage tariff: USD 0.04 per 1M INPUT tokens, output not billed (docs.fastino.ai/pricing, GLiDE row; page read 22 Oct 2026 20:34 UTC, saved copy sha256 22c0758f). Counted under interpretation I-1 (a manufacturer's standard, non-promotional list price counts from day 1) by the frozen pricing module pricing_v15 (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7), which carries one added launch-list-price key for this row and nothing else. Applied to this row's own provider-reported input tokens on the answered rows; the 17 requests that returned an error are priced at the API's 65,536-token prompt ceiling, the documented context estimate. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common items of this draw, leaving out the 15 very long items that count for no row; this row scored 1,497 items, so the basis here is 1,482 items. This row: USD 0.023065 per 1,000 decisions (0.357x the official cost cap USD 0.064595), against USD 0.049101 over all 1,497 scored items. The older measurement of this vendor's GLiDE on 6 Oct 2026 (key fastino-glide) was priced at the then tariff of USD 0.15 per 1M input tokens and is unchanged.",
    "common_basis": {
     "n_items": 1482,
     "rule": "common",
     "usd_per_1000_all_items": 0.0491012959251837
    },
    "kind": "mixed",
    "usage_estimated_rows": 17,
    "usd_per_1000": 0.023065479082321188
   },
   "custody_exclusion": {
    "derived_gold_sha256": "9f1f8200d0a1b4384f43ffc0a91f3f36e4790fc3dbcef7486bf233b0142baa47",
    "exclusion_list_sha256": "31b47af58dce134a70e5cbf60aad40db0000dd398f26649070fb7c9dfd1deeb3",
    "items": 3,
    "rule": "an item a provider has already seen is never scored for that provider; the 3 items were never sent and are not in this row's gold",
    "scored_items": 1497
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 2,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1497,
    "treatment": "share"
   },
   "display": "Fastino GLiDE (hosted API, 22 Oct 2026)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.fastino.ai /v1/systemone API, model `fastino/GLiDE` (responses report glide_v2), evaluator key, every request with store:false; latency measured from Sandy (Germany) on every item without API adjustment. Full S+P draw of v1.6-regular-20261012 minus the 3 custody-excluded items (1,497 requests, one pass, no retries) through the unchanged run_api_full_v161.py; the full-set exposure gate logged the 1,197 sealed items to the provider before the first request. Measured 22 Oct 2026 by the evaluator; the vendor did not take part in the run.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 79.78345749455909,
    "I_sealed": 72.35339980704964,
    "base": 76.06842865080436,
    "excess": -2.4094690828462646,
    "gap": 7.430057687509446,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 90.51495356571994,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 86.78829820698331,
       "judge": 96.044825313118,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 75.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 70,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 72.88717875232587,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.7719731384208,
       "hard": 72.51880498670339,
       "judge": 82.85006684010338,
       "standard": 59.2371969588571
      }
     },
     "sealed|choice": {
      "cc": 86.92305256837759,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 81.78559037816582,
       "judge": 87.65915768854063,
       "standard": 90.50833157561695
      }
     },
     "sealed|noul": {
      "cc": 62.1766990537663,
      "n": {
       "easy": 58,
       "hard": 109,
       "judge": 64,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 50.458715596330286,
       "judge": 65.625,
       "standard": 70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 67.96044779900504,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 66,
       "standard": 75
      },
      "tiers": {
       "easy": 86.59605824557663,
       "hard": 62.546614906669795,
       "judge": 68.82118654512506,
       "standard": 68.17920024120974
      }
     }
    }
   },
   "jevbench_score": 76.44111138489434,
   "key": "fastino-glide-api",
   "last_measured_on": "2026-10-22",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.40",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:https://fastino.ai#fastino-glide-api",
   "noul_decisive": {
    "acc_among_decisive": 0.9749216300940439,
    "decisive": 319,
    "decisive_rate": 0.8621621621621621,
    "n": 373,
    "supported": true,
    "valid": 370
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 2,
    "axes": {
     "calibration": 93.01106628088438,
     "cost": 59.111125613009825,
     "intelligence": 76.39198423827783,
     "speed": 87.30503336667806
    },
    "board_release": "v1.6.43",
    "capability": 84.7015252595811,
    "capability_api_board_rank": 2,
    "capability_open_board_rank": null,
    "capability_rank": 4,
    "composite_ci95": {
     "A": [
      75.0560545503276,
      77.30340620880787
     ],
     "B": [
      74.53280727316283,
      77.76808823284206
     ],
     "C": [
      75.0560545503276,
      77.30340620880787
     ]
    },
    "jevbench_score": 76.60902426459442,
    "measured_in": "v1.6.40",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 2,
    "ranks": {
     "A": 2,
     "B": 2,
     "C": 2,
     "capability": 9
    },
    "scores": {
     "A": 76.60902426459442,
     "B": 76.56551765292035,
     "C": 76.60902426459442
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.3601445150561631,
     "p50_s_raw": 0.3601445150561631,
     "p95_s_adjusted": 0.5164399832952765,
     "p95_s_raw": 0.5164399832952765
    },
    "views": {
     "Balanced 33:33:33": 72.35582674096251,
     "Emphasis on Accuracy 60:20:20": 73.91800482728986,
     "Emphasis on Cost 20:20:60": 66.40429071449509,
     "Emphasis on Speed 20:60:20": 77.67599908573227,
     "Intelligence only": 76.39198423827783
    }
   },
   "rank": 8,
   "ranked": true,
   "ranks": {
    "A": 8,
    "B": 5,
    "C": 7,
    "capability": 11
   },
   "repo": "https://fastino.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 76.44111138489434,
    "B": 76.36628298199761,
    "C": 76.44111138489434
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1497,
    "p50_s_adjusted": 0.35695503698661923,
    "p50_s_raw": 0.35695503698661923,
    "p95_s_adjusted": 0.5584349319804464,
    "p95_s_raw": 0.5584349319804464
   },
   "status": {
    "answered_ok": 1480,
    "missing": 0,
    "rows": 1497,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API (POST api.fastino.ai/v1/systemone, model fastino/GLiDE) through the unchanged TypeSafe adapter with one added request field, store:false; no vendor code ran locally. SCORED ON 1,497 OF THE DRAW'S 1,500 ITEMS: 3 items Fastino had already seen through its hosted GLiNER-2.5-Decide on 8 Oct were never sent (custody rule: an item a provider has seen is never scored for that provider). The row is scored with the unmodified frozen scorer on the draw's gold without those 3 items (a second invocation of the same scorer; the other rows of this board are scored on all 1,500 items). PROVIDER-HOSTED: Fastino's API serves the model as `glide_v2` (every answered response reported it); the weights and serving were not inspected, and this is not the open-weights GLiDE 1.2 release, which is a separate pending measurement. RETENTION: Fastino's default is to retain API inputs; store:false on every request; the vendor's not-stored response header was enforced on every answered request by the run harness (error responses carry no such header by the vendor's documentation). NOT ANSWERED, counted wrong: 15 items exceed the API's 65,536-token prompt limit (HTTP 400 'prompt exceeds the maximum context length'), counted wrong; and 2 gateway errors (HTTP 502) from Fastino's API, not retried (one pass), counted wrong. PRICE CHANGED SINCE THE FIRST MEASUREMENT: on 6 Oct 2026 this vendor's GLiDE was listed at USD 0.15 per 1M input tokens with a 40,000-token prompt limit and scored on 600 items (key fastino-glide, Capability 77.0, outside the cost cap); today's tariff is USD 0.04 per 1M input tokens with a 65,536-token limit, and this row is a new measurement on the full draw. The older row stays as history. Latency was measured from Sandy (Germany) with no API adjustment, while the evaluation host was busy with other work; 2 answered decisions took more than 10 seconds (21.15 s and 76.42 s); v1.6 applies no 10-second rule. Native probabilities on all three question types, no generated tokens; the customer-side statements about the model are the vendor's and are not independently verified. SEALED ITEMS SENT TO THE VENDOR: the API sees the 1,197 sealed items it was sent; acceptable because every sealed item of this draw was already retired from scoring.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": -5.961080915580955,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": -209.54990941340955
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.7920792079207921,
       "cc": 61.30216821922519,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 43.99999999999999,
        "score": 65.35719394717714
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.30216821922519
      },
      "composite_gen": {
       "accuracy": 0.9108910891089109,
       "cc": 85.67892146745031,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 85.18518518518519,
        "score": 75.10361173749092
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.67892146745031
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.29471891787144,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 98.88415675361429
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.29471891787144
      },
      "jaggedness": {
       "accuracy": 0.7944444444444444,
       "cc": 58.69250954076001,
       "cc_by_type": {
        "choice": 79.78825794032724,
        "noul": 18.918918918918926,
        "score": 77.37035176303387
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 58.69250954076001
      },
      "llm_router": {
       "accuracy": 0.952076677316294,
       "cc": 88.04795205712485,
       "cc_by_type": {
        "choice": 96.83014880690322,
        "noul": 100,
        "score": 67.31370736447127
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 88.04795205712485
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7916666666666666,
       "cc": 52.713148360991475,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 33.33333333333333,
        "score": 49.548379790878194
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 6,
        "score": 7
       },
       "score": 52.713148360991475
      },
      "multilingual": {
       "accuracy": 0.8218181818181818,
       "cc": 69.62686351470224,
       "cc_by_type": {
        "choice": 85.32768177372026,
        "noul": 55.55555555555556,
        "score": 67.99735321483094
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.62686351470224
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 74.54014406808,
       "cc_by_type": {
        "choice": 100,
        "noul": 73.33333333333334,
        "score": 50.28709887090663
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.54014406808
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 81.53200714105311,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 80,
        "score": 79.6110364381744
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 81.53200714105311
      },
      "usecase": {
       "accuracy": 0.8636363636363636,
       "cc": 74.04969523790491,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 58.33333333333333,
        "score": 73.61967394900884
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 74.04969523790491
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 49.28337524952835,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50,
        "score": 16.897744796204094
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 49.28337524952835
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 55.50357736254705,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 33.17739875430783
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 55.50357736254705
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 93.40496888083449,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 100,
        "score": 88.17950841241498
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 93.40496888083449
      },
      "el": {
       "accuracy": 0.9333333333333333,
       "cc": 94.3834224733954,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100,
        "score": 99.91004395649901
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 94.3834224733954
      },
      "en": {
       "accuracy": 0.8740225890529973,
       "cc": 76.08356533618894,
       "cc_by_type": {
        "choice": 88.57714697160152,
        "noul": 68.34532374100719,
        "score": 71.3282252959581
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 76.08356533618894
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 74.21236753974422,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 32.425030841092365
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 74.21236753974422
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.72593685668032,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.84447723670762
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.72593685668032
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 92.06735412018907,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 97.37853294880247
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 92.06735412018907
      },
      "hi": {
       "accuracy": 0.8888888888888888,
       "cc": 70.34650149976363,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 66.66666666666667,
        "score": 44.37283783262424
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 4
       },
       "score": 70.34650149976363
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 66.41366853069502,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 13.27062367470393
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 66.41366853069502
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 47.24100841291983,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 19.999999999999996,
        "score": 36.020086398410015
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 47.24100841291983
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 18.987371364068355,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 56.96211409220508
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 18.987371364068355
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.96955130238082,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.9086539071425
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.96955130238082
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 1.7442176532610603,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": -35.45998773285752
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 1.7442176532610603
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 78.64726523260191,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80,
        "score": 71.17666662459372
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 78.64726523260191
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 78.0409563759746,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 65.9437294697034
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 78.0409563759746
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 59.78378036796139,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 44.8832559975012
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 59.78378036796139
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 85.72572454543065,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 78.91630407107458
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 85.72572454543065
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 83.93118510075651,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 84.05161981839854
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 83.93118510075651
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 65.20564880948768,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 62.28361309512973
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 65.20564880948768
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.9066666666666666,
       "cc": 87.9446602499091,
       "cc_by_type": {
        "choice": 87.9446602499091
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 87.9446602499091
      },
      "noul": {
       "accuracy": 0.8337801608579088,
       "cc": 66.75603217158177,
       "cc_by_type": {
        "noul": 66.75603217158177
       },
       "low_n": false,
       "n": 373,
       "n_by_type": {
        "noul": 373
       },
       "score": 66.75603217158177
      },
      "score": {
       "accuracy": 0.8074866310160428,
       "cc": 69.47514714474733,
       "cc_by_type": {
        "score": 69.47514714474733
       },
       "low_n": false,
       "n": 374,
       "n_by_type": {
        "score": 374
       },
       "score": 69.47514714474733
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 28.27943132511149,
       "cc_by_type": {
        "choice": 68,
        "noul": -25,
        "score": 41.83829397533446
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 28.27943132511149
      },
      "code_linting": {
       "accuracy": 1,
       "cc": 93.28648102165937,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 79.8594430649781
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 93.28648102165937
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 64.14435699593669,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 24.953396191062115
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 64.14435699593669
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -4.4326718070937305,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 33.33333333333333,
        "score": -88.15181659087183
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.875,
       "cc": 68.1764684457371,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 85.68882562706641
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 68.1764684457371
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 56.88352721922263,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": 49.89586467653584
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 56.88352721922263
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 51.09999092184478,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -14.28571428571429,
        "score": 67.58568705124861
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 51.09999092184478
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 52.69076696339014,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 58.07230089017044
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 52.69076696339014
      },
      "insurance_claims": {
       "accuracy": 0.6428571428571429,
       "cc": 62.65779285636237,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 50,
        "score": 58.80671190242044
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 4
       },
       "score": 62.65779285636237
      },
      "knowledge_graphs": {
       "accuracy": 0.9545454545454546,
       "cc": 90.65854177937764,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 71.97562533813296
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 90.65854177937764
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 67.60857584215633,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 58.89807894765766
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.60857584215633
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 45.5354720458297,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 67.78921183641384
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 45.5354720458297
      },
      "llm_guardrails": {
       "accuracy": 0.9629629629629629,
       "cc": 80.95439830812018,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 82.8631949243605
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 80.95439830812018
      },
      "model_routing": {
       "accuracy": 0.9351851851851852,
       "cc": 84.8410274166256,
       "cc_by_type": {
        "choice": 96.49350649350649,
        "noul": 100,
        "score": 58.02957575637029
       },
       "low_n": false,
       "n": 216,
       "n_by_type": {
        "choice": 101,
        "noul": 65,
        "score": 50
       },
       "score": 84.8410274166256
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 55.15593532195087,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 88.50043552439388
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 55.15593532195087
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 67.57303995857968,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 69.38578654240574
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 67.57303995857968
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 62.24567525530753,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 36.7370257659226
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 62.24567525530753
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 56.083255830130874,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 63.901941403436105
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 56.083255830130874
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.92021847495123,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.76065542485364
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.92021847495123
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.84523935751662,
      93.09512159193915
     ],
     "I": [
      71.97632052544036,
      79.51320538269066
     ],
     "capability": [
      81.08718032572618,
      85.69953649578142
     ],
     "composite": {
      "A": [
       74.90033635084451,
       77.08874894746072
      ],
      "B": [
       74.3293214235386,
       77.56901714437397
      ],
      "C": [
       74.90033635084451,
       77.08874894746072
      ]
     },
     "seed": 16
    },
    "ci95_n": 1197,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1497,
    "n_public": 300,
    "n_sealed_side": 1197,
    "on_S": {
     "calibration": 92.84353248115252,
     "capability": 84.45598056597845,
     "intelligence": 76.06842865080436
    },
    "per_type": {
     "choice": {
      "accuracy": 0.9066666666666666,
      "calibration": 91.30791369573681,
      "cc": 87.9446602499091,
      "cc_by_type": {
       "choice": 87.9446602499091
      },
      "cc_tiered": 88.71900306704876,
      "cc_tiered_open": 90.51495356571994,
      "cc_tiered_sealed": 86.92305256837759,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 87.9446602499091
     },
     "noul": {
      "accuracy": 0.8337801608579088,
      "calibration": 92.12139262649261,
      "cc": 66.75603217158177,
      "cc_by_type": {
       "noul": 66.75603217158177
      },
      "cc_tiered": 69.0624696096989,
      "cc_tiered_open": 75.94824016563147,
      "cc_tiered_sealed": 62.1766990537663,
      "low_n": false,
      "n": 373,
      "n_by_type": {
       "noul": 373
      },
      "score": 66.75603217158177
     },
     "score": {
      "accuracy": 0.8074866310160428,
      "calibration": 95.10129112122816,
      "cc": 69.47514714474733,
      "cc_by_type": {
       "score": 69.47514714474733
      },
      "cc_tiered": 70.42381327566545,
      "cc_tiered_open": 72.88717875232587,
      "cc_tiered_sealed": 67.96044779900504,
      "low_n": false,
      "n": 374,
      "n_by_type": {
       "score": 374
      },
      "score": 69.47514714474733
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.016000000000000014,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.008042895442359255,
     "n": 373
    },
    "score": {
     "invalid_rate": 0.010695187165775444,
     "n": 374
    }
   },
   "views": {
    "Balanced 33:33:33": 72.18991121677152,
    "Emphasis on Accuracy 60:20:20": 73.69286891912549,
    "Emphasis on Cost 20:20:60": 66.32035800171175,
    "Emphasis on Speed 20:60:20": 77.46597567840699,
    "Intelligence only": 76.06842865080436
   }
  },
  {
   "addendum": null,
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "Fastino",
   "axes": {
    "calibration": 62.13288114639033,
    "cost": 45.81558630529788,
    "intelligence": 7.011351364482595,
    "speed": 84.98447762423095
   },
   "calibration": {
    "parts": {
     "choice": {
      "basis": "provider_confidence_only",
      "ece_score": 28.328881967263143,
      "n": 305,
      "n_ece": 305,
      "score": 28.328881967263143,
      "score_before_time_limit": 28.328881967263143,
      "time_limit_failed_share": 0,
      "top_label_ece": 0.3583555901636843
     },
     "noul": {
      "basis": "provider_confidence_only",
      "ece_score": 76.72217340082736,
      "n": 370,
      "n_ece": 370,
      "score": 76.72217340082736,
      "score_before_time_limit": 76.72217340082736,
      "time_limit_failed_share": 0,
      "top_label_ece": 0.11638913299586323
     },
     "score": {
      "basis": "provider_confidence_only",
      "ece_score": 81.3475880710805,
      "n": 366,
      "n_ece": 366,
      "score": 81.3475880710805,
      "score_before_time_limit": 81.3475880710805,
      "time_limit_failed_share": 0,
      "top_label_ece": 0.09326205964459748
     }
    },
    "score": 62.13288114639033
   },
   "capability": 34.57211625543646,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     0.10778599179976782,
     1.0396621480462107
    ],
    "B": [
     0.07441782713955666,
     0.785906717748602
    ],
    "C": [
     0.07485138319428321,
     0.7219876028098686
    ]
   },
   "cost": {
    "basis": "ESTIMATE: published Fastino sources conflict (USD 0.03/M vs 0.15/M input); conservative higher USD 0.15/M input, 0/M output x measured tokens; no invoice",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.07366349999999999
    },
    "kind": "mixed",
    "usage_estimated_rows": 15,
    "usd_per_1000": 0.06399515910629654
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Fastino GLiNER-2.5-Decide (hosted API)",
   "endpoint_condition": "external hosted API (Fastino); received the full set: 1,200 sealed + 300 public items (v1.6.1)",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 6.2445368778313695,
    "I_sealed": 7.778165851133819,
    "base": 7.011351364482595,
    "excess": -4.107016615740694,
    "gap": -1.5336289733024495,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 18.73361063349411,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 18.270270270270274,
       "hard": 15.06763133060712,
       "judge": 24.851680949241928,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -21.739130434782606,
       "judge": 10.000000000000009,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -36.36363636363635,
       "hard": -28.70370370370372,
       "judge": -16.949152542372904,
       "standard": -99.99999999999996
      }
     },
     "sealed|choice": {
      "cc": 15.115509018204584,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 25.504397309881035,
       "hard": 8.880090497737543,
       "judge": 18.06331471135942,
       "standard": 17.970193373568186
      }
     },
     "sealed|noul": {
      "cc": 8.218988535196875,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 4.0000000000000036,
       "hard": -6.796116504854366,
       "judge": 7.8947368421052655,
       "standard": 40.8450704225352
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -38.29787234042554,
       "hard": -44.60784313725492,
       "judge": -24.55089820359284,
       "standard": -63.451776649746215
      }
     }
    }
   },
   "jevbench_score": 0.3433992147410167,
   "key": "fastino-gliner-2-5-decide",
   "last_measured_on": "2026-10-05",
   "licence": "proprietary hosted API",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "fastino/GLiNER-2.5-Decide (requested model id)",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5405405405405406,
    "decisive_rate": 1,
    "supported": true,
    "valid": 370
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 44,
    "axes": {
     "calibration": 62.13288114639033,
     "cost": 45.81558630529788,
     "intelligence": 7.011351364482595,
     "speed": 84.7052606179839
    },
    "board_release": "v1.6.43",
    "capability": 34.57211625543646,
    "capability_api_board_rank": 28,
    "capability_open_board_rank": null,
    "capability_rank": 151,
    "composite_ci95": {
     "A": [
      0.10777031922591683,
      1.0393833813480395
     ],
     "B": [
      0.07441185007614143,
      0.7857792694163073
     ],
     "C": [
      0.07484049946244224,
      0.7217940148250275
     ]
    },
    "jevbench_score": 0.34332996901846624,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 208,
    "ranks": {
     "A": 208,
     "B": 208,
     "C": 208,
     "capability": 218
    },
    "scores": {
     "A": 0.34332996901846624,
     "B": 0.24643533434670511,
     "C": 0.23842358959615717
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.4409300498664379,
     "p50_s_raw": 0.4409300498664379,
     "p95_s_adjusted": 0.7675456959754229,
     "p95_s_raw": 0.7675456959754229
    },
    "views": {
     "Balanced 33:33:33": 0.2810101668608893,
     "Emphasis on Accuracy 60:20:20": 0.17887060474062133,
     "Emphasis on Cost 20:20:60": 0.37538092966437175,
     "Emphasis on Speed 20:60:20": 0.41302239566745963,
     "Intelligence only": 0.11575817604301195
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": 249,
   "ranked": true,
   "ranks": {
    "A": 249,
    "B": 249,
    "C": 249,
    "capability": 259
   },
   "repo": "https://fastino.ai/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.3433992147410167,
    "B": 0.2464638726515236,
    "C": 0.23847167690348384
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.42045173048973083,
    "p50_s_raw": 0.42045173048973083,
    "p95_s_adjusted": 0.7548073027282952,
    "p95_s_raw": 0.7548073027282952
   },
   "status": {
    "answered_ok": 1469,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "confidence",
    "noul": "confidence",
    "score": "confidence"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      55.437755987711,
      66.14780229160992
     ],
     "I": [
      4.625975407661763,
      10.579291699427245
     ],
     "capability": [
      30.439944046543882,
      37.65484132935661
     ],
     "composite": {
      "A": [
       0.10778599179976782,
       1.0396621480462107
      ],
      "B": [
       0.07441782713955666,
       0.785906717748602
      ],
      "C": [
       0.07485138319428321,
       0.7219876028098686
      ]
     },
     "seed": 38
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 62.13288114639033,
     "capability": 34.57211625543646,
     "intelligence": 7.011351364482595
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3426666666666667,
      "calibration": 28.328881967263143,
      "cc": 15.936455007909936,
      "cc_by_type": {
       "choice": 15.936455007909936
      },
      "cc_tiered": 16.924559825849347,
      "cc_tiered_open": 18.73361063349411,
      "cc_tiered_sealed": 15.115509018204584,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 15.936455007909936
     },
     "noul": {
      "accuracy": 0.5333333333333333,
      "calibration": 76.72217340082736,
      "cc": 6.666666666666665,
      "cc_by_type": {
       "noul": 6.666666666666665
      },
      "cc_tiered": 4.109494267598437,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 8.218988535196875,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 6.666666666666665
     },
     "score": {
      "accuracy": 0.248,
      "calibration": 81.3475880710805,
      "cc": -42.280701754385944,
      "cc_by_type": {
       "score": -42.280701754385944
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.022666666666666613,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.02400000000000002,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.2810720197974522,
    "Emphasis on Accuracy 60:20:20": 0.17888563915023908,
    "Emphasis on Cost 20:20:60": 0.3754471502388532,
    "Emphasis on Speed 20:60:20": 0.4132629943808129,
    "Intelligence only": 0.11575817604301195
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 599,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Google",
   "axes": {
    "calibration": 60.93,
    "cost": 29.17,
    "intelligence": 58.57,
    "speed": 78.98
   },
   "calibration": {
    "score": 60.93
   },
   "capability": 59.75,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     15.19,
     17.76
    ],
    "B": [
     15.24,
     18.64
    ],
    "C": [
     10.55,
     17.76
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.25/M input, 1.5/M output (spec gemini-3.1-flash-lite.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.22953722504230117
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Gemini 3.1 Flash-Lite",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 63.88,
    "I_sealed": 48.12,
    "base": 58.57,
    "per_type_split": {
     "open|choice": {
      "cc": 70.9158,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.3243,
       "hard": 73.5766,
       "judge": 72.3138,
       "standard": 54.7928
      }
     },
     "open|noul": {
      "cc": 62.9917,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 65.2174,
       "judge": 50,
       "standard": 66.6667
      }
     },
     "open|score": {
      "cc": 57.7334,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 43.1818,
       "hard": 58.3333,
       "judge": 56.0508,
       "standard": 66.3333
      }
     },
     "sealed|choice": {
      "cc": 66.1467,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 83.3179,
       "hard": 49.5853,
       "judge": 72.8643,
       "standard": 80.6076
      }
     },
     "sealed|noul": {
      "cc": 56.3289,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 45.4545,
       "judge": 60,
       "standard": 62.5
      }
     },
     "sealed|score": {
      "cc": 21.8863,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 54.2222,
       "hard": 21.553,
       "judge": -22.9545,
       "standard": 73.6463
      }
     }
    }
   },
   "jevbench_score": 16.93,
   "key": "gemini-3.1-flash-lite",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 37,
    "axes": {
     "calibration": 60.93,
     "cost": 29.17,
     "intelligence": 58.57,
     "speed": 79.03
    },
    "board_release": "v1.6.43",
    "capability": 59.75,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      15.19,
      17.77
     ],
     "B": [
      15.24,
      18.65
     ],
     "C": [
      10.55,
      17.77
     ]
    },
    "jevbench_score": 16.94,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 148,
    "ranks": {
     "A": 148,
     "B": 139,
     "C": 132,
     "capability": 134
    },
    "scores": {
     "A": 16.94,
     "B": 17.46,
     "C": 16.14
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 599,
     "p50_s_adjusted": 0.957,
     "p50_s_raw": 0.957,
     "p95_s_adjusted": 1.306,
     "p95_s_raw": 1.306
    },
    "views": null
   },
   "rank": 195,
   "ranked": true,
   "ranks": {
    "A": 195,
    "B": 186,
    "C": 182,
    "capability": 166
   },
   "repo": "https://benchmarkheaven.com/jev-models/gemini-3.1-flash-lite",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.93,
    "B": 17.46,
    "C": 16.14
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.962,
    "p50_s_raw": 0.962,
    "p95_s_adjusted": 1.317,
    "p95_s_raw": 1.317
   },
   "status": {
    "answered_ok": 599,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "96adf8183eaf9826b99e72a821469fa1fcc61783bc7b04ec2a95f0abcea883e8"
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "AutoTrust AI Lab",
   "axes": {
    "calibration": 90.33483359631919,
    "cost": 72.5645152590109,
    "intelligence": 49.817428608862244,
    "speed": 89.61965681094298
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05524139109810591,
      "ece_score": 88.95172178037882,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.95172178037882,
      "score_before_time_limit": 88.95172178037882,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.30188679245283023,
      "brier": 0.091757346419259,
      "ece": 0.03659159632329119,
      "n": 371,
      "score": 92.68168073534177,
      "score_before_time_limit": 92.68168073534177,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9728997289972899,
      "interval90_width": 0.4146341463414633,
      "n": 369,
      "nrps": 0.07505075554087659,
      "score": 89.37109827323698,
      "score_before_time_limit": 89.37109827323698,
      "time_limit_failed_share": 0,
      "top_ece": 0.06876363949719191
     }
    },
    "score": 90.33483359631919
   },
   "capability": 70.07613110259072,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 694,051 input + 1,477 output tokens over 1,477 decisions = 470.905890318 tokens/decision. GPU-time cost USD 0.00821327109943156/1,000; old list-price cost USD 0.0425915301286391/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00821327109943156/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.04193846
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.008213271099431558
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "GEV-26B-Decide (AutoTrust, Gemma-4-26B-A4B + LoRA + head)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 51.11414911073161,
    "I_sealed": 48.520708106992885,
    "base": 49.817428608862244,
    "excess": 0.02005336130048141,
    "gap": 2.593441003738725,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.91276210170015,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 69.80182447310474,
       "judge": 72.31377719182596,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 35.146307798481715,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 10.000000000000009,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 47.28337743201297,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.41806192777556,
       "hard": 49.795958510853765,
       "judge": 44.087055938179034,
       "standard": 39.48535526720095
      }
     },
     "sealed|choice": {
      "cc": 69.65586334909833,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 64.36651583710406,
       "judge": 69.27374301675978,
       "standard": 75.61276019214189
      }
     },
     "sealed|noul": {
      "cc": 27.311750523580937,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 63.99999999999999,
       "hard": 8.737864077669899,
       "judge": 28.947368421052634,
       "standard": 43.66197183098593
      }
     },
     "sealed|score": {
      "cc": 48.59451044829939,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 67.07336665308927,
       "hard": 39.678983511563985,
       "judge": 58.83673780283293,
       "standard": 41.82279518757497
      }
     }
    }
   },
   "jevbench_score": 70.80379734355978,
   "key": "gev-26b-decide",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "autotrust/GEV-26B-Decide@7c89590ead085bf77630b4bf68264ea30b6ddc78",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9613899613899614,
    "decisive_rate": 0.6981132075471698,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.33483359631919,
     "cost": 51.12030271996105,
     "intelligence": 49.817428608862244,
     "speed": 89.87310530072351
    },
    "board_release": "v1.6.43",
    "capability": 70.07613110259072,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 26,
    "capability_rank": 46,
    "composite_ci95": {
     "A": [
      48.122003834420376,
      66.28949313101391
     ],
     "B": [
      44.53704625593728,
      63.445630864100984
     ],
     "C": [
      33.418058218347475,
      54.44178965547935
     ]
    },
    "jevbench_score": 64.2201412904201,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 29,
    "rank": 50,
    "ranks": {
     "A": 50,
     "B": 50,
     "C": 58,
     "capability": 78
    },
    "scores": {
     "A": 64.2201412904201,
     "B": 60.60132222808652,
     "C": 44.59732034056951
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2535779389785603,
     "p50_s_raw": 0.051788969489280134,
     "p95_s_adjusted": 0.40604858579754366,
     "p95_s_raw": 0.12802429289877182
    },
    "views": {
     "Balanced 33:33:33": 58.6687573959346,
     "Emphasis on Accuracy 60:20:20": 54.59949882434838,
     "Emphasis on Cost 20:20:60": 55.22102328110309,
     "Emphasis on Speed 20:60:20": 67.97953354893902,
     "Intelligence only": 49.45428333114776
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 69,
   "ranked": true,
   "ranks": {
    "A": 69,
    "B": 93,
    "C": 103,
    "capability": 99
   },
   "repo": "https://huggingface.co/autotrust/GEV-26B-Decide",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.80379734355978,
    "B": 65.17643740642315,
    "C": 49.169303710805394
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.258882079977775,
    "p50_s_raw": 0.054441039988887496,
    "p95_s_adjusted": 0.4216308011615183,
    "p95_s_raw": 0.13581540058075917
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-26B-A4B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.06256442860612,
      90.45126283775208
     ],
     "I": [
      44.10305960817391,
      54.39355915396382
     ],
     "capability": [
      65.77131123789238,
      71.56937054169023
     ],
     "composite": null,
     "seed": 42
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.33483359631919,
     "capability": 70.07613110259072,
     "intelligence": 49.817428608862244
    },
    "per_type": {
     "choice": {
      "accuracy": 0.772,
      "calibration": 88.95172178037882,
      "cc": 70.84205640233793,
      "cc_by_type": {
       "choice": 70.84205640233793
      },
      "cc_tiered": 70.28431272539925,
      "cc_tiered_open": 70.91276210170015,
      "cc_tiered_sealed": 69.65586334909833,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.84205640233793
     },
     "noul": {
      "accuracy": 0.664,
      "calibration": 92.68168073534177,
      "cc": 32.800000000000004,
      "cc_by_type": {
       "noul": 32.800000000000004
      },
      "cc_tiered": 31.229029161031328,
      "cc_tiered_open": 35.146307798481715,
      "cc_tiered_sealed": 27.311750523580937,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 32.800000000000004
     },
     "score": {
      "accuracy": 0.656,
      "calibration": 89.37109827323698,
      "cc": 48.08438271156575,
      "cc_by_type": {
       "score": 48.08438271156575
      },
      "cc_tiered": 47.93894394015618,
      "cc_tiered_open": 47.28337743201297,
      "cc_tiered_sealed": 48.59451044829939,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 48.08438271156575
     }
    },
    "raw_on_A": {
     "calibration": 88.68291663446959,
     "capability": 72.10556374473956,
     "intelligence": 55.528210855009505
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.16246281055172,
    "Emphasis on Accuracy 60:20:20": 58.28570864113504,
    "Emphasis on Cost 20:20:60": 68.3929047533248,
    "Emphasis on Speed 20:60:20": 73.7209395158067,
    "Intelligence only": 49.45428333114776
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Goddragon (Hugging Face)",
   "axes": {
    "calibration": 77.19519806916665,
    "cost": 80.74092951874186,
    "intelligence": 27.391250053452495,
    "speed": 92.05594930781893
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12453810152460322,
      "ece_score": 75.09237969507936,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 75.09237969507936,
      "score_before_time_limit": 75.09237969507936,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3512064343163539,
      "brier": 0.1726437405208735,
      "ece": 0.11296355354302531,
      "n": 373,
      "score": 77.40728929139493,
      "score_before_time_limit": 77.40728929139493,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8790322580645161,
      "interval90_width": 0.48521505376344093,
      "n": 372,
      "nrps": 0.12849056705545897,
      "score": 79.08592522102566,
      "score_before_time_limit": 79.08592522102566,
      "time_limit_failed_share": 0,
      "top_ece": 0.1448954642620138
     }
    },
    "score": 77.19519806916665
   },
   "capability": 52.29322406130957,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 662,682 input + 0 output tokens over 1,485 decisions = 446.250505051 tokens/decision. GPU-time cost USD 0.00438499412707729/1,000; old list-price cost USD 0.0133875151515152/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00438499412707729/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.013253640000000002
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004384994127077292
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Arbiter-4B (Goddragon, NeoHorse 4B pointer model, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA A100-SXM4 80 GB (FIN-02, Finland, EU, pay-as-you-go non-spot, USD 2.168/h), shared by the two NeoHorse-family rows as two separate back-to-back uses (sealed input deleted from the node between them). The author's own server.py as shipped on stock PyPI vLLM 0.30.0 / torch 2.13.0 in a pre-built image, container with --network none (cut proven from inside before the sealed input), Hub offline, every credential unset, read-only bundle, all weight shards sha256-verified; driven by the unchanged pinned evaluator driver, one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 21 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 32.1215764916093,
    "I_sealed": 22.660923615295694,
    "base": 27.391250053452495,
    "excess": -0.37887389404210836,
    "gap": 9.460652876313603,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 58.95334556281439,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 47.15319282793332,
       "judge": 56.49307844429796,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -21.739130434782606,
       "judge": -9.999999999999998,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 37.41138391201351,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.07472571843502,
       "hard": 42.643741131070314,
       "judge": 32.30337526144111,
       "standard": 24.277011546547755
      }
     },
     "sealed|choice": {
      "cc": 46.87143166012293,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 69.50444726810673,
       "hard": 40.67306580316872,
       "judge": 48.87365328109698,
       "standard": 44.94832313857837
      }
     },
     "sealed|noul": {
      "cc": 4.770267303463389,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 34.48275862068966,
       "hard": 1.8181818181818077,
       "judge": -16.92307692307692,
       "standard": 28.358208955223873
      }
     },
     "sealed|score": {
      "cc": 16.341071882300767,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 34.31316593972345,
       "hard": 7.707251567970719,
       "judge": 10.049632979159162,
       "standard": 34.059823836961925
      }
     }
    }
   },
   "jevbench_score": 16.509995434698638,
   "key": "goddragon-arbiter-4b",
   "last_measured_on": "2026-10-21",
   "licence": "Apache-2.0 (HF metadata and LICENSE), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.36",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Goddragon/Arbiter-4B@8fd20568f359f4156de55f016027efdc841d690a",
   "noul_decisive": {
    "acc_among_decisive": 0.8429752066115702,
    "decisive": 242,
    "decisive_rate": 0.6487935656836461,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.19519806916665,
     "cost": 66.19900073922123,
     "intelligence": 27.391250053452495,
     "speed": 92.16491010646149
    },
    "board_release": "v1.6.43",
    "capability": 52.29322406130957,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 89,
    "capability_rank": 115,
    "composite_ci95": {
     "A": [
      9.862003577191583,
      22.969397756248135
     ],
     "B": [
      8.069131709312094,
      19.832903146488743
     ],
     "C": [
      6.848613595271931,
      15.950970664061202
     ]
    },
    "jevbench_score": 15.917217056211483,
    "measured_in": "v1.6.36",
    "method": "v1.6",
    "open_board_rank": 117,
    "rank": 155,
    "ranks": {
     "A": 155,
     "B": 158,
     "C": 155,
     "capability": 170
    },
    "scores": {
     "A": 15.917217056211483,
     "B": 13.406692451289457,
     "C": 11.053622955702421
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24398728700034553,
     "p50_s_raw": 0.046993643500172766,
     "p95_s_adjusted": 0.24896700299996155,
     "p95_s_raw": 0.04948350149998077
    },
    "views": {
     "Balanced 33:33:33": 14.413667848999275,
     "Emphasis on Accuracy 60:20:20": 11.075877822934517,
     "Emphasis on Cost 20:20:60": 16.19147002227193,
     "Emphasis on Speed 20:60:20": 17.828945195366323,
     "Intelligence only": 8.22044918523227
    }
   },
   "rank": 197,
   "ranked": true,
   "ranks": {
    "A": 197,
    "B": 201,
    "C": 197,
    "capability": 206
   },
   "repo": "https://huggingface.co/Goddragon/Arbiter-4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.509995434698638,
    "B": 13.73908219778341,
    "C": 11.465274607429611
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24426879499983442,
    "p50_s_raw": 0.04713439749991721,
    "p95_s_adjusted": 0.2549981790999709,
    "p95_s_raw": 0.05249908954998545
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author server vllm-fast/server.py at 8fd20568f359f4156de55f016027efdc841d690a, started directly as shipped (independent static review PASS WITH FINDINGS, 0 blockers; no subprocess/network in the server import path; run_evaluation.py, evaluate.py and provenance/* never started). Author defaults kept: position pooling ON (its shipped default), no prefix or head cache. BASE: the repository's backbone/config.json modification notice states that the language-model weights were fine-tuned from Qwen/Qwen3.5-4B (tensor values unchanged by repackaging), and the card's base_model field names the NeoHorse lineage (TokenRhythm/NeoHorse-Jev-4B); the base string Qwen/Qwen3.5-4B is used because it is the upstream model the repository itself states and the only one the frozen pricing reference can resolve. Lineage claims are the authors' and unverified (the manifest names only a local path). FIRST MEASUREMENT (free-queue submission 574B4820). CONTEXT CAP: the author server refuses requests whose state exceeds 8,192 tokens or whose state+question row exceeds 12,288 tokens (HTTP 422); on this draw that is exactly the 15 items of 76k-81k tokens, pre-registered by id and position; they count as wrong, like d3-turbo, elf-1-12b and seb-9b on this draw. SAME FAMILY: Goddragon/Arbiter-4B and roudone/Decider-4B are two different checkpoints submitted through the free queue that both serve as 'NeoHorse-Jev-4B' (TYPESAFE_MODEL alias set for the frozen adapter). PROVENANCE, author statements (unverified): base lineage TokenRhythm/NeoHorse-Jev-4B / Qwen3.5-4B; training on the typed-decisions train split plus classification sets, TRAM, HelpSteer2 and constructed examples (README). No rule against public items was found in the reviewed code. HARDWARE: A100 80 GB (the RTX PRO 6000 was out of stock); latency reflects that card. Shipped temperature 1.1942 is the author's choice (their own held-out ECE got worse with it).",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5454545454545454,
       "cc": 13.1573417948352,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": 100,
        "score": -93.86130794882774
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 13.1573417948352
      },
      "calibration_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 6.759864864914857,
       "cc_by_type": {
        "choice": 30.010604453870634,
        "noul": -12,
        "score": 2.268990140873939
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 6.759864864914857
      },
      "composite_gen": {
       "accuracy": 0.40594059405940597,
       "cc": -11.562426232861531,
       "cc_by_type": {
        "choice": 54.47154471544716,
        "noul": -48.14814814814815,
        "score": -41.0106752658836
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.6875,
       "cc": 42.72141227498651,
       "cc_by_type": {
        "choice": 100,
        "noul": -19.999999999999996,
        "score": 48.16423682495953
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 42.72141227498651
      },
      "jaggedness": {
       "accuracy": 0.37777777777777777,
       "cc": 6.9964087476416905,
       "cc_by_type": {
        "choice": 14.82194417709337,
        "noul": -13.513513513513509,
        "score": 19.68079557934521
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 6.9964087476416905
      },
      "llm_router": {
       "accuracy": 0.7635782747603834,
       "cc": 65.55835613406727,
       "cc_by_type": {
        "choice": 61.169322884564586,
        "noul": 79.38144329896907,
        "score": 56.124302218668156
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 65.55835613406727
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2222222222222222,
       "cc": -30.50844844645876,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -50,
        "score": -42.55627317442783
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5818181818181818,
       "cc": 25.353468409646783,
       "cc_by_type": {
        "choice": 58.9175089664167,
        "noul": -19.444444444444443,
        "score": 36.58734070696811
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 25.353468409646783
      },
      "negation_pair_gen": {
       "accuracy": 0.4883720930232558,
       "cc": -8.074110162754854,
       "cc_by_type": {
        "choice": 53.333333333333336,
        "noul": -33.333333333333336,
        "score": -44.22233048826456
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.625,
       "cc": 52.06391478029295,
       "cc_by_type": {
        "choice": 52.06391478029295
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 52.06391478029295
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 65.2669271494847,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 60.00000000000001,
        "score": 65.83081147848414
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 65.2669271494847
      },
      "usecase": {
       "accuracy": 0.5181818181818182,
       "cc": 26.957730500631396,
       "cc_by_type": {
        "choice": 50.98039215686277,
        "noul": 0,
        "score": 29.892799345031406
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 26.957730500631396
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.1875,
       "cc": -38.476656738814235,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": -50,
        "score": -70.19187497834747
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 56.66171289152569,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 36.65180534124374
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 56.66171289152569
      },
      "de": {
       "accuracy": 0.71875,
       "cc": 65.80687892506546,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 75,
        "score": 70.20824739466542
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 65.80687892506546
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 23.999811413863814,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 88.94543051719666
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 23.999811413863814
      },
      "en": {
       "accuracy": 0.5490877497827976,
       "cc": 26.586689973573968,
       "cc_by_type": {
        "choice": 47.059469618383936,
        "noul": 12.230215827338121,
        "score": 20.470384474999847
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 26.586689973573968
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 58.68217680179415,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": -4.377613150898152
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 58.68217680179415
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 14.608201601683332,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 9.944823384284973
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 14.608201601683332
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 45.62934066592817,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 58.06449258601979
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 45.62934066592817
      },
      "hi": {
       "accuracy": 0.38095238095238093,
       "cc": -23.312194873205982,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -50,
        "score": -69.65725501067939
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 75.85729665962164,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 55.63112614410264
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 75.85729665962164
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 20.6313226795196,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": -19.999999999999996,
        "score": 53.37927383681138
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 20.6313226795196
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -8.161926992907732,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 19.95866346572126
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 72.54315187844138,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 84.29612230199078
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 72.54315187844138
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -6.316997789459312,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 7.0230326056480274
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5641025641025641,
       "cc": 20.232627552011937,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -40,
        "score": 31.16762450961178
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 20.232627552011937
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 49.75002180092513,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 31.677879191814885
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 49.75002180092513
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 2.1082100916838846,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 28.45228984951973
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 2.1082100916838846
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 29.533493664601565,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50,
        "score": -17.92125813663008
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 29.533493664601565
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 62.240877851683614,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 51.238762587308884
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 62.240877851683614
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 19.184165055251437,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": -24.93338054046037
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 19.184165055251437
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.608,
       "cc": 49.36757304961825,
       "cc_by_type": {
        "choice": 49.36757304961825
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 49.36757304961825
      },
      "noul": {
       "accuracy": 0.544,
       "cc": 8.800000000000008,
       "cc_by_type": {
        "noul": 8.800000000000008
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 8.800000000000008
      },
      "score": {
       "accuracy": 0.472,
       "cc": 22.702016645247348,
       "cc_by_type": {
        "score": 22.702016645247348
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 22.702016645247348
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -0.8162936667601267,
       "cc_by_type": {
        "choice": 52,
        "noul": -50,
        "score": -4.44888100028038
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": 1.8108502810784488,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": 11.111111111111116,
        "score": -12.51085840452173
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 1.8108502810784488
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 45.134859392293265,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 0,
        "score": 51.6647407785058
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 45.134859392293265
      },
      "demand_forecasting": {
       "accuracy": 0.11764705882352941,
       "cc": -38.24161193978049,
       "cc_by_type": {
        "choice": -28.65497076023392,
        "noul": -100,
        "score": 13.930134940892447
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 19.891098676055922,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 21.9921366078779
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 19.891098676055922
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": 16.539907187567064,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": -66.66666666666667,
        "score": 54.02223728597164
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 16.539907187567064
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 18.7768282505638,
       "cc_by_type": {
        "choice": 41.17647058823529,
        "noul": -14.28571428571429,
        "score": 29.4397284491704
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 18.7768282505638
      },
      "gaming": {
       "accuracy": 0.22727272727272727,
       "cc": -19.139393677670046,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100,
        "score": 26.2055123467808
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 21.22332456763144,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -19.999999999999996,
        "score": 46.16997370289432
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 21.22332456763144
      },
      "knowledge_graphs": {
       "accuracy": 0.2727272727272727,
       "cc": 9.518799890413932,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": 33.33333333333333,
        "score": -1.9932077520272928
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 9.518799890413932
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 20.02190004982577,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": 5.026940459554852
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 20.02190004982577
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -25.972993526020257,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": -47.811453696340344
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 28.56631129583379,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 40.24787822531327
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 28.56631129583379
      },
      "model_routing": {
       "accuracy": 0.7926267281105991,
       "cc": 66.41479171839013,
       "cc_by_type": {
        "choice": 74.28571428571429,
        "noul": 84.84848484848484,
        "score": 40.11017602097127
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.41479171839013
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 4.953745673457386,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100,
        "score": 72.44281091672534
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 4.953745673457386
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 49.47362762453108,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 48.513733384271085
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 49.47362762453108
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 3.6459962938239614,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50,
        "score": -24.283685995375404
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 3.6459962938239614
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 12.90172875431424,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 15.516780465841274
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 12.90172875431424
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 77.49722676858175,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 82.49168030574525
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 77.49722676858175
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.25547879639667,
      80.42268223181145
     ],
     "I": [
      22.680931701137304,
      31.747668356672595
     ],
     "capability": [
      47.99634931098826,
      55.30745997453868
     ],
     "composite": null,
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.19519806916665,
     "capability": 52.29322406130957,
     "intelligence": 27.391250053452495
    },
    "per_type": {
     "choice": {
      "accuracy": 0.608,
      "calibration": 75.09237969507936,
      "cc": 49.36757304961825,
      "cc_by_type": {
       "choice": 49.36757304961825
      },
      "cc_tiered": 52.91238861146866,
      "cc_tiered_open": 58.95334556281439,
      "cc_tiered_sealed": 46.87143166012293,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.36757304961825
     },
     "noul": {
      "accuracy": 0.544,
      "calibration": 77.40728929139493,
      "cc": 8.800000000000008,
      "cc_by_type": {
       "noul": 8.800000000000008
      },
      "cc_tiered": 2.3851336517316946,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 4.770267303463389,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 8.800000000000008
     },
     "score": {
      "accuracy": 0.472,
      "calibration": 79.08592522102566,
      "cc": 22.702016645247348,
      "cc_by_type": {
       "score": 22.702016645247348
      },
      "cc_tiered": 26.87622789715714,
      "cc_tiered_open": 37.41138391201351,
      "cc_tiered_sealed": 16.341071882300767,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 22.702016645247348
     }
    },
    "raw_on_A": {
     "calibration": 74.72248487244359,
     "capability": 51.694359455861786,
     "intelligence": 28.666234039279978
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.066818031101212,
    "Emphasis on Accuracy 60:20:20": 11.30176558419029,
    "Emphasis on Cost 20:20:60": 17.752480551544775,
    "Emphasis on Speed 20:60:20": 18.415820457112464,
    "Intelligence only": 8.22044918523227
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 600,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "OpenAI",
   "axes": {
    "calibration": 94.31,
    "cost": 30.16,
    "intelligence": 92.97,
    "speed": 73.21
   },
   "calibration": {
    "score": 94.31
   },
   "capability": 93.64,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     20.98,
     21.75
    ],
    "B": [
     22.55,
     23.65
    ],
    "C": [
     20.98,
     21.75
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.2/M input, 1.2/M output (spec gpt-5.6-luna.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.2127451776649746
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 4,
    "n_over_limit_answered_ok": 4,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "GPT-5.6 Luna (low reasoning effort)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 91.05,
    "I_sealed": 89.75,
    "base": 92.97,
    "per_type_split": {
     "open|choice": {
      "cc": 93.8551,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100,
       "hard": 94.3378,
       "judge": 92.0897,
       "standard": 92.4655
      }
     },
     "open|noul": {
      "cc": 100,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 100,
       "judge": 100,
       "standard": 100
      }
     },
     "open|score": {
      "cc": 79.3051,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 77.1818,
       "hard": 94.0625,
       "judge": 83.7819,
       "standard": 44.137
      }
     },
     "sealed|choice": {
      "cc": 89.5553,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 94.4393,
       "hard": 82.5487,
       "judge": 95.4774,
       "standard": 92.2431
      }
     },
     "sealed|noul": {
      "cc": 94.6471,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 100,
       "judge": 90,
       "standard": 100
      }
     },
     "sealed|score": {
      "cc": 85.0542,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 90.2472,
       "hard": 78.3022,
       "judge": 91.2727,
       "standard": 86.6341
      }
     }
    }
   },
   "jevbench_score": 21.36,
   "key": "gpt-5.6-luna",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 34,
    "axes": {
     "calibration": 94.91,
     "cost": 30.16,
     "intelligence": 94.07,
     "speed": 73.12
    },
    "board_release": "v1.6.43",
    "capability": 94.49,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      21.08,
      21.77
     ],
     "B": [
      22.67,
      23.67
     ],
     "C": [
      21.08,
      21.77
     ]
    },
    "jevbench_score": 21.41,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 121,
    "ranks": {
     "A": 121,
     "B": 111,
     "C": 103,
     "capability": 5
    },
    "scores": {
     "A": 21.41,
     "B": 23.14,
     "C": 21.41
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 600,
     "p50_s_adjusted": 1.588,
     "p50_s_raw": 1.588,
     "p95_s_adjusted": 3.067,
     "p95_s_raw": 3.067
    },
    "views": null
   },
   "rank": 182,
   "ranked": true,
   "ranks": {
    "A": 182,
    "B": 174,
    "C": 166,
    "capability": 3
   },
   "repo": "https://benchmarkheaven.com/jev-models/gpt-5.6-luna",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.36,
    "B": 23.06,
    "C": 21.36
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.586,
    "p50_s_raw": 1.586,
    "p95_s_adjusted": 3.011,
    "p95_s_raw": 3.011
   },
   "status": {
    "answered_ok": 600,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "5d37f643d12df8d42d02c44dc364058832ddc8298187ebc522f43050b9199cac"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 596,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "OpenAI",
   "axes": {
    "calibration": 99.92,
    "cost": 38.41,
    "intelligence": 96.88,
    "speed": 71.38
   },
   "calibration": {
    "score": 99.92
   },
   "capability": 98.4,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     38.57,
     39.32
    ],
    "B": [
     40.95,
     42.13
    ],
    "C": [
     38.57,
     39.32
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.1/M input, 0.5/M output (spec gpt-6-luna.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.1129575296108291
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 5,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "GPT-6 Luna (default medium reasoning effort)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 95.37,
    "I_sealed": 93.25,
    "base": 96.88,
    "per_type_split": {
     "open|choice": {
      "cc": 99.245,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100,
       "hard": 98.1126,
       "judge": 100,
       "standard": 100
      }
     },
     "open|noul": {
      "cc": 97.1429,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.4286,
       "hard": 100,
       "judge": 100,
       "standard": 100
      }
     },
     "open|score": {
      "cc": 89.716,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 88.4545,
       "hard": 100,
       "judge": 93.5932,
       "standard": 63.963
      }
     },
     "sealed|choice": {
      "cc": 99.2244,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 100,
       "hard": 98.061,
       "judge": 100,
       "standard": 100
      }
     },
     "sealed|noul": {
      "cc": 93.3636,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 100,
       "hard": 90.9091,
       "judge": 90,
       "standard": 100
      }
     },
     "sealed|score": {
      "cc": 87.1594,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 98.5972,
       "hard": 78.3133,
       "judge": 89.5,
       "standard": 95.622
      }
     }
    }
   },
   "jevbench_score": 39.1,
   "key": "gpt-6-luna",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "unknown",
   "previous": {
    "api_board_rank": 30,
    "axes": {
     "calibration": 100,
     "cost": 38.41,
     "intelligence": 96.88,
     "speed": 71.77
    },
    "board_release": "v1.6.43",
    "capability": 98.44,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      38.68,
      39.37
     ],
     "B": [
      41.06,
      42.18
     ],
     "C": [
      38.68,
      39.37
     ]
    },
    "jevbench_score": 39.16,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 84,
    "ranks": {
     "A": 84,
     "B": 72,
     "C": 66,
     "capability": 1
    },
    "scores": {
     "A": 39.16,
     "B": 41.79,
     "C": 39.16
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 596,
     "p50_s_adjusted": 1.824,
     "p50_s_raw": 1.824,
     "p95_s_adjusted": 3.646,
     "p95_s_raw": 3.646
    },
    "views": null
   },
   "rank": 151,
   "ranked": true,
   "ranks": {
    "A": 151,
    "B": 131,
    "C": 122,
    "capability": 1
   },
   "repo": "https://benchmarkheaven.com/jev-models/gpt-6-luna",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 39.1,
    "B": 41.74,
    "C": 39.1
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.932,
    "p50_s_raw": 1.932,
    "p95_s_adjusted": 3.765,
    "p95_s_raw": 3.765
   },
   "status": {
    "answered_ok": 596,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "36dce9d9c3ec86a404450d2d6b5e9d6f57a229474e17c5a78a5805689d481088"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 600,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "OpenAI",
   "axes": {
    "calibration": 98.48,
    "cost": 38.98,
    "intelligence": 95.66,
    "speed": 71.82
   },
   "calibration": {
    "score": 98.48
   },
   "capability": 97.07,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     39.82,
     40.81
    ],
    "B": [
     42.21,
     43.68
    ],
    "C": [
     39.82,
     40.81
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.1/M input, 0.5/M output (spec gpt-6-luna-low.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.10814365482233503
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 3,
    "n_over_limit_answered_ok": 3,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "GPT-6 Luna (low reasoning effort)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 93.06,
    "I_sealed": 93.11,
    "base": 95.66,
    "per_type_split": {
     "open|choice": {
      "cc": 96.9832,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100,
       "hard": 96.2252,
       "judge": 100,
       "standard": 92.4655
      }
     },
     "open|noul": {
      "cc": 96.3492,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 100,
       "judge": 100,
       "standard": 88.8889
      }
     },
     "open|score": {
      "cc": 85.8511,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 87.0909,
       "hard": 99.6667,
       "judge": 94.1525,
       "standard": 45.1481
      }
     },
     "sealed|choice": {
      "cc": 100,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 100,
       "hard": 100,
       "judge": 100,
       "standard": 100
      }
     },
     "sealed|noul": {
      "cc": 92.1872,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 88.2353,
       "hard": 90.9091,
       "judge": 90,
       "standard": 100
      }
     },
     "sealed|score": {
      "cc": 87.1364,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 97.8611,
       "hard": 78.3133,
       "judge": 88.8182,
       "standard": 96.8976
      }
     }
    }
   },
   "jevbench_score": 40.39,
   "key": "gpt-6-luna-low",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "unknown",
   "previous": {
    "api_board_rank": 28,
    "axes": {
     "calibration": 99.02,
     "cost": 38.98,
     "intelligence": 96.62,
     "speed": 71.66
    },
    "board_release": "v1.6.43",
    "capability": 97.82,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      39.96,
      40.78
     ],
     "B": [
      42.4,
      43.66
     ],
     "C": [
      39.96,
      40.78
     ]
    },
    "jevbench_score": 40.48,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 81,
    "ranks": {
     "A": 81,
     "B": 70,
     "C": 64,
     "capability": 3
    },
    "scores": {
     "A": 40.48,
     "B": 43.16,
     "C": 40.48
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 600,
     "p50_s_adjusted": 1.854,
     "p50_s_raw": 1.854,
     "p95_s_adjusted": 3.683,
     "p95_s_raw": 3.683
    },
    "views": null
   },
   "rank": 145,
   "ranked": true,
   "ranks": {
    "A": 145,
    "B": 128,
    "C": 118,
    "capability": 2
   },
   "repo": "https://benchmarkheaven.com/jev-models/gpt-6-luna-low",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 40.39,
    "B": 43.02,
    "C": 40.39
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.896,
    "p50_s_raw": 1.896,
    "p95_s_adjusted": 3.465,
    "p95_s_raw": 3.465
   },
   "status": {
    "answered_ok": 600,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "c5fa6729c53fbe4fc0ec88af75ee26aa997274bc1ef4a820a8c468616c433e33"
   }
  },
  {
   "adapter_id": "images/gte-reranker-modernbert-base/spec.json:reranker_adapter.RerankerAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Alibaba-NLP",
   "axes": {
    "calibration": 83.4556233651092,
    "cost": 69.26965431492843,
    "intelligence": 0,
    "speed": 93.50537133277697
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0790764439307023,
      "ece_score": 84.18471121385954,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 84.18471121385954,
      "score_before_time_limit": 84.18471121385954,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 1,
      "brier": 0.25231019651494707,
      "ece": 0.08962639509020176,
      "n": 375,
      "score": 82.07472098195964,
      "score_before_time_limit": 82.07472098195964,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 1,
      "interval90_width": 1,
      "n": 375,
      "nrps": 0.17667560258952839,
      "score": 84.10743789950841,
      "score_before_time_limit": 84.10743789950841,
      "time_limit_failed_share": 0,
      "top_ece": 0.07058781971015167
     }
    },
    "score": 83.4556233651092
   },
   "capability": 41.7278116825546,
   "class": "reranker",
   "composite_ci95": {
    "A": [
     0,
     0.0036625510569188633
    ],
    "B": [
     0,
     0.002344053723577124
    ],
    "C": [
     0,
     0.0025434382339714324
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:thenlper/gte-base); no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.010576570197044334
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Alibaba GTE Reranker ModernBERT-base",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 0,
    "I_sealed": 0,
    "base": 0,
    "excess": -2.573387642438244,
    "gap": 0,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 0,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": -3.8062283737024116,
       "judge": -6.789716545814108,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -1.920002214440264,
       "hard": -1.3526386292960213,
       "judge": 1.2639659476201492,
       "standard": 0.3612908498087819
      }
     },
     "sealed|choice": {
      "cc": 0,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -5.092010937846398,
       "hard": -6.391402714932147,
       "judge": -5.214152700186197,
       "standard": -7.525557334647108
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -1.317182445372178,
       "hard": 0.03511410774166279,
       "judge": -0.2802076730282099,
       "standard": 0.30690766431314964
      }
     }
    }
   },
   "jevbench_score": 0,
   "key": "gte-reranker-modernbert-base",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive_rate": 0,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.4556233651092,
     "cost": 69.26965431492843,
     "intelligence": 0,
     "speed": 93.76120791185616
    },
    "board_release": "v1.6.43",
    "capability": 41.7278116825546,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 110,
    "capability_rank": 138,
    "composite_ci95": {
     "A": [
      0,
      0.003662687483805933
     ],
     "B": [
      0,
      0.0023440984279851505
     ],
     "C": [
      0,
      0.002543532974865231
     ]
    },
    "jevbench_score": 0,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 195,
    "rank": 239,
    "ranks": {
     "A": 239,
     "B": 239,
     "C": 239,
     "capability": 202
    },
    "scores": {
     "A": 0,
     "B": 0,
     "C": 0
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19835810186341404,
     "p50_s_raw": 0.024179050931707025,
     "p95_s_adjusted": 0.2120455999393016,
     "p95_s_raw": 0.031022799969650804
    },
    "views": {
     "Balanced 33:33:33": 0,
     "Emphasis on Accuracy 60:20:20": 0,
     "Emphasis on Cost 20:20:60": 0,
     "Emphasis on Speed 20:60:20": 0,
     "Intelligence only": 0
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 280,
   "ranked": true,
   "ranks": {
    "A": 280,
    "B": 280,
    "C": 280,
    "capability": 243
   },
   "repo": "https://huggingface.co/Alibaba-NLP/gte-reranker-modernbert-base",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0,
    "B": 0,
    "C": 0
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19940753942355513,
    "p50_s_raw": 0.024703769711777568,
    "p95_s_adjusted": 0.2237284903414547,
    "p95_s_raw": 0.03686424517072736
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "149M ModernBERT cross-encoder",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.59536793549519,
      87.51314141306226
     ],
     "I": [
      0,
      1.339161261347401
     ],
     "capability": [
      39.372795544944715,
      43.89015023558699
     ],
     "composite": {
      "A": [
       0,
       0.0036625510569188633
      ],
      "B": [
       0,
       0.002344053723577124
      ],
      "C": [
       0,
       0.0025434382339714324
      ]
     },
     "seed": 39
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.4556233651092,
     "capability": 41.7278116825546,
     "intelligence": 0
    },
    "per_type": {
     "choice": {
      "accuracy": 0.17866666666666667,
      "calibration": 84.18471121385954,
      "cc": -5.036802667601379,
      "cc_by_type": {
       "choice": -5.036802667601379
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0
     },
     "noul": {
      "accuracy": 0,
      "calibration": 82.07472098195964,
      "cc": -100,
      "cc_by_type": {
       "noul": -100
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.304,
      "calibration": 84.10743789950841,
      "cc": -0.24811073820258933,
      "cc_by_type": {
       "score": -0.24811073820258933
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 88.93979403901187,
     "capability": 44.469897019505936,
     "intelligence": 0
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0,
    "Emphasis on Accuracy 60:20:20": 0,
    "Emphasis on Cost 20:20:60": 0,
    "Emphasis on Speed 20:60:20": 0,
    "Intelligence only": 0
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Omar / @kouhxp (hi@kouh.me)",
   "axes": {
    "calibration": 75.89600269702092,
    "cost": 84.91099734772837,
    "intelligence": 8.600978834309654,
    "speed": 71.56789504904872
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08639168454258676,
      "ece_score": 82.72166309148264,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 81.50841203280756,
      "score_before_time_limit": 82.72166309148264,
      "time_limit_failed_share": 0.014666666666666666
     },
     "noul": {
      "abstention_rate": 0.706043956043956,
      "brier": 0.24853405620583507,
      "ece": 0.12309816483516484,
      "n": 364,
      "score": 73.1692096,
      "score_before_time_limit": 75.38036703296703,
      "time_limit_failed_share": 0.029333333333333333
     },
     "score": {
      "interval90_coverage": 0.8695652173913043,
      "interval90_width": 0.6397192028985506,
      "n": 368,
      "nrps": 0.16768646620403746,
      "score": 73.01038645825521,
      "score_before_time_limit": 74.39917098327638,
      "time_limit_failed_share": 0.018666666666666668,
      "top_ece": 0.17216505706521745
     }
    },
    "score": 75.89600269702092
   },
   "capability": 42.24849076566529,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.8B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 789,980 input + 0 output tokens over 1,500 decisions = 526.653333333 tokens/decision. GPU-time cost USD 0.00318395378306878/1,000; old list-price cost USD 0.00526653333333333/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00318395378306878/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0031839537830687836
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 29,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Gutsy 0.8B v0.3",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium pod (2xRTX3090 node, Germany), CPU-only by the author's own config (n_gpu_layers 0), 32 vCPU, egress cut with iptables and proven before the sealed input was uploaded; the author's own gutsy-inference 0.3.0 with their models.example.json settings unchanged (n_ctx 8192, reject_slot true, n_threads null)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 9.070301583187167,
    "I_sealed": 8.131656085432141,
    "base": 8.600978834309654,
    "excess": -9.134599083584687,
    "gap": 0.9386454977550258,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 25.412389147780978,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 37.72972972972974,
       "hard": 20.729789241899976,
       "judge": 12.986156888595916,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -47.82608695652174,
       "judge": -60,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 1.7985156017805268,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.240515151515154,
       "hard": -7.74469814814811,
       "judge": 1.9498728813559296,
       "standard": -2.5630925925926196
      }
     },
     "sealed|choice": {
      "cc": 24.394968256296423,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 23.036813014912926,
       "hard": 20.853028470924478,
       "judge": 26.031150119222776,
       "standard": 29.70365265334253
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": -43.58974358974359,
       "hard": -78.78787878787878,
       "judge": -47.22222222222222,
       "standard": -61.40350877192983
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 12.014685981308393,
       "hard": -2.484919999999935,
       "judge": -12.305056637168143,
       "standard": 12.255661711711696
      }
     }
    }
   },
   "jevbench_score": 0.7626893562712093,
   "key": "gutsy-0.8b-v0.3",
   "last_measured_on": "2026-10-10",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "kouhxp/gutsy@d325eb1742cc1c200e04b24417f192a0dbf74f91",
   "noul_decisive": {
    "acc_among_decisive": 0.7383177570093458,
    "decisive": 107,
    "decisive_rate": 0.29395604395604397,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.50040036924202,
     "cost": 78.3542548810428,
     "intelligence": 8.600978834309654,
     "speed": 72.8877498600171
    },
    "board_release": "v1.6.43",
    "capability": 43.05068960177584,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.35718114749129054,
      1.8750127120910802
     ],
     "B": [
      0.2489508080898962,
      1.3935931889582263
     ],
     "C": [
      0.24804246353561837,
      1.3020921611743612
     ]
    },
    "jevbench_score": 0.7604367004496922,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 158,
    "rank": 201,
    "ranks": {
     "A": 201,
     "B": 201,
     "C": 201,
     "capability": 196
    },
    "scores": {
     "A": 0.7604367004496922,
     "B": 0.5441131491342173,
     "C": 0.5280810419789529
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.6613200119999418,
     "p50_s_raw": 0.755660005999971,
     "p95_s_adjusted": 3.0957915995999423,
     "p95_s_raw": 1.4728957997999712
    },
    "views": {
     "Balanced 33:33:33": 0.6218800588368827,
     "Emphasis on Accuracy 60:20:20": 0.39424898336691744,
     "Emphasis on Cost 20:20:60": 0.8792467571253542,
     "Emphasis on Speed 20:60:20": 0.8693565458528719,
     "Intelligence only": 0.25450928339078893
    }
   },
   "rank": 242,
   "ranked": true,
   "ranks": {
    "A": 242,
    "B": 242,
    "C": 242,
    "capability": 242
   },
   "repo": "https://huggingface.co/kouhxp/gutsy",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.7626893562712093,
    "B": 0.5450346322764674,
    "C": 0.5296453862994508
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.901271966999866,
    "p50_s_raw": 0.875635983499933,
    "p95_s_adjusted": 3.6657787574997194,
    "p95_s_raw": 1.7578893787498597
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5862068965517241,
       "cc": 30.238494444444445,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 0,
        "score": 32.38215000000001
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 30.238494444444445
      },
      "calibration_gen": {
       "accuracy": 0.38392857142857145,
       "cc": -19.57648651457238,
       "cc_by_type": {
        "choice": 41.770186335403736,
        "noul": -92.3076923076923,
        "score": -8.191953571428567
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.2653061224489796,
       "cc": -24.789154706790118,
       "cc_by_type": {
        "choice": 29.62962962962963,
        "noul": -100,
        "score": -3.9970937499999915
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.5555555555555556,
       "cc": 21.924596601307183,
       "cc_by_type": {
        "choice": 52.94117647058823,
        "noul": 0,
        "score": 12.83261333333332
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 21.924596601307183
      },
      "jaggedness": {
       "accuracy": 0.2756756756756757,
       "cc": -23.766898037764125,
       "cc_by_type": {
        "choice": 9.680407596109312,
        "noul": -79.48717948717949,
        "score": -1.4939222222221993
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.4583333333333333,
       "cc": 21.153469908954868,
       "cc_by_type": {
        "choice": 26.61402693915468,
        "noul": 1.3698630136986356,
        "score": 35.47651977401129
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 21.153469908954868
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.25806451612903225,
       "cc": -38.646075919732446,
       "cc_by_type": {
        "choice": 26.153846153846153,
        "noul": -100,
        "score": -42.09207391304348
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.346031746031746,
       "cc": -10.63069831663855,
       "cc_by_type": {
        "choice": 30.138826690550832,
        "noul": -58.62068965517242,
        "score": -3.4102319852940655
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.26785714285714285,
       "cc": -26.649703571428574,
       "cc_by_type": {
        "choice": 14.285714285714286,
        "noul": -90,
        "score": -4.234825000000009
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.17543859649122806,
       "cc": -4.328523862375131,
       "cc_by_type": {
        "choice": -4.328523862375131
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 0
      },
      "perturb_pair": {
       "accuracy": 0.5166666666666667,
       "cc": 19.74922901184277,
       "cc_by_type": {
        "choice": 41.02228047182176,
        "noul": -28.57142857142857,
        "score": 46.79683513513512
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 19.74922901184277
      },
      "usecase": {
       "accuracy": 0.3206751054852321,
       "cc": -16.990773799131347,
       "cc_by_type": {
        "choice": 27.153594141453095,
        "noul": -75.43859649122807,
        "score": -2.6873190476190745
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.36363636363636365,
       "cc": -27.022149999999996,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": -50,
        "score": -51.06644999999999
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5,
       "cc": -12.711864406779661,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.20833333333333334,
       "cc": -32.610591814595644,
       "cc_by_type": {
        "choice": 5.325443786982243,
        "noul": -100,
        "score": -3.1572192307691926
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.4666666666666667,
       "cc": 2.2274148960461893,
       "cc_by_type": {
        "choice": 40.25347012673505,
        "noul": -33.333333333333336,
        "score": -0.23789210526314886
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 2.2274148960461893
      },
      "el": {
       "accuracy": 0.2222222222222222,
       "cc": -51.29191646586346,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -100,
        "score": -69.53840000000002
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.34405719392314565,
       "cc": -11.632396312202536,
       "cc_by_type": {
        "choice": 22.602355580482268,
        "noul": -61.654135338345874,
        "score": 4.154590821255999
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.5,
       "cc": 13.68897878787879,
       "cc_by_type": {
        "choice": 60.72727272727273,
        "noul": -27.27272727272727,
        "score": 7.612390909090916
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 13.68897878787879
      },
      "fi": {
       "accuracy": 0.2222222222222222,
       "cc": -42.25352112676057,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": -100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.3333333333333333,
       "cc": -15.758361508496193,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": -100,
        "score": -21.887214285714272
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.3,
       "cc": -16.665771829710142,
       "cc_by_type": {
        "choice": 13.043478260869566,
        "noul": -60,
        "score": -3.0407937499999926
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0
      },
      "id": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.6470588235294118,
       "cc": 27.99738442348378,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 0,
        "score": 34.76789375
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 27.99738442348378
      },
      "ja": {
       "accuracy": 0.3125,
       "cc": -23.373968770567163,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": -71.42857142857143,
        "score": -17.39252187499998
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.2727272727272727,
       "cc": -31.130543497267748,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": -27.81785999999997
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.36363636363636365,
       "cc": -3.6486553113553115,
       "cc_by_type": {
        "choice": 34.06593406593406,
        "noul": -50,
        "score": 4.988100000000006
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.46153846153846156,
       "cc": -0.1424452553389107,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 0,
        "score": -50.288060000000016
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.42857142857142855,
       "cc": 0.5464480874316955,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0.5464480874316955
      },
      "pl": {
       "accuracy": 0.4,
       "cc": 12.299376097900057,
       "cc_by_type": {
        "choice": 60.64713598600787,
        "noul": -19.999999999999996,
        "score": -3.7490076923077043
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 12.299376097900057
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 26.514546072026793,
       "cc_by_type": {
        "choice": 45.7286432160804,
        "noul": 0,
        "score": 33.81499499999999
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 26.514546072026793
      },
      "sv": {
       "accuracy": 0,
       "cc": -56.24097777777778,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": -35.3896
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.45454545454545453,
       "cc": 0.7387181286549614,
       "cc_by_type": {
        "choice": 47.368421052631575,
        "noul": 33.33333333333333,
        "score": -78.48560000000002
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0.7387181286549614
      },
      "uk": {
       "accuracy": 0.09090909090909091,
       "cc": -46.02425072463768,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": -100,
        "score": -29.3771
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.125,
       "cc": -50.39295435435435,
       "cc_by_type": {
        "choice": -29.729729729729726,
        "noul": -33.333333333333336,
        "score": -88.11580000000001
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.41733333333333333,
       "cc": 24.512005527724998,
       "cc_by_type": {
        "choice": 24.512005527724998
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 24.512005527724998
      },
      "noul": {
       "accuracy": 0.21066666666666667,
       "cc": -57.86666666666667,
       "cc_by_type": {
        "noul": -57.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.3466666666666667,
       "cc": 0.9913860770578653,
       "cc_by_type": {
        "score": 0.9913860770578653
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.9913860770578653
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3333333333333333,
       "cc": -20.792305967417175,
       "cc_by_type": {
        "choice": 30.807248764415156,
        "noul": -60,
        "score": -33.184166666666684
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.16666666666666666,
       "cc": -51.04649872817785,
       "cc_by_type": {
        "choice": -8.01963993453355,
        "noul": -60,
        "score": -85.11985625
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.4,
       "cc": 7.8360928866832085,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50,
        "score": 38.02440769230768
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 7.8360928866832085
      },
      "demand_forecasting": {
       "accuracy": 0.23076923076923078,
       "cc": -39.86741871729342,
       "cc_by_type": {
        "choice": -8.43373493975904,
        "noul": -100,
        "score": -11.168521212121219
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2777777777777778,
       "cc": -17.362241654722098,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100,
        "score": 3.8138961538461302
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.1724137931034483,
       "cc": -29.604258430458433,
       "cc_by_type": {
        "choice": -2.564102564102566,
        "noul": -60,
        "score": -26.248672727272737
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.38461538461538464,
       "cc": -8.077263014492752,
       "cc_by_type": {
        "choice": 47.82608695652174,
        "noul": -100,
        "score": 27.942124
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.27586206896551724,
       "cc": -33.71579672167217,
       "cc_by_type": {
        "choice": 16.831683168316832,
        "noul": -100,
        "score": -17.97907333333333
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.14285714285714285,
       "cc": -70.40589841269842,
       "cc_by_type": {
        "choice": 4.761904761904763,
        "noul": -100,
        "score": -115.9796
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.21739130434782608,
       "cc": -16.350229066353496,
       "cc_by_type": {
        "choice": -9.923664122137408,
        "noul": -60,
        "score": 20.872976923076926
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.36,
       "cc": -2.498773089430904,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -75,
        "score": -16.23615666666669
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.38461538461538464,
       "cc": -26.543207810993252,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": -14.055852941176482
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.3793103448275862,
       "cc": -5.703711032698185,
       "cc_by_type": {
        "choice": 44.95412844036697,
        "noul": -50,
        "score": -12.065261538461524
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.5,
       "cc": 20.346316406896474,
       "cc_by_type": {
        "choice": 42.106673813990895,
        "noul": 9.090909090909083,
        "score": 9.841366315789447
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 20.346316406896474
      },
      "moderation_trust_safety": {
       "accuracy": 0.3125,
       "cc": -21.78233634085213,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100,
        "score": -2.1891142857142887
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.38095238095238093,
       "cc": -7.3411204741657015,
       "cc_by_type": {
        "choice": 54.77386934673366,
        "noul": -100,
        "score": 23.20276923076924
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.5384615384615384,
       "cc": 28.47671559224318,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 33.33333333333333,
        "score": -10.167337500000006
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 28.47671559224318
      },
      "scientific_discovery": {
       "accuracy": 0.21875,
       "cc": -30.3334952412572,
       "cc_by_type": {
        "choice": 12.350597609561751,
        "noul": -100,
        "score": -3.35108333333336
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.18181818181818182,
       "cc": -40.61265147175815,
       "cc_by_type": {
        "choice": 14.08114558472553,
        "noul": -100,
        "score": -35.91909999999998
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      69.67939224880561,
      78.1660305687077
     ],
     "I": [
      6.552688842251461,
      11.980624454535823
     ],
     "capability": [
      38.73036369799716,
      44.464298745938066
     ],
     "composite": null,
     "seed": 21
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 75.89600269702092,
     "capability": 42.24849076566529,
     "intelligence": 8.600978834309654
    },
    "per_type": {
     "choice": {
      "accuracy": 0.41733333333333333,
      "calibration": 81.50841203280756,
      "cc": 24.512005527724998,
      "cc_by_type": {
       "choice": 24.512005527724998
      },
      "cc_tiered": 24.9036787020387,
      "cc_tiered_open": 25.412389147780978,
      "cc_tiered_sealed": 24.394968256296423,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 24.512005527724998
     },
     "noul": {
      "accuracy": 0.21066666666666667,
      "calibration": 73.1692096,
      "cc": -57.86666666666667,
      "cc_by_type": {
       "noul": -57.86666666666667
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.3466666666666667,
      "calibration": 73.01038645825521,
      "cc": 0.9913860770578653,
      "cc_by_type": {
       "score": 0.9913860770578653
      },
      "cc_tiered": 0.8992578008902634,
      "cc_tiered_open": 1.7985156017805268,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.9913860770578653
     }
    },
    "raw_on_A": {
     "calibration": 77.32547272800478,
     "capability": 42.983248163593274,
     "intelligence": 8.641023599181757
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.6250876092152238,
    "Emphasis on Accuracy 60:20:20": 0.39502000557661987,
    "Emphasis on Cost 20:20:60": 0.8936035905573986,
    "Emphasis on Speed 20:60:20": 0.870514864639876,
    "Intelligence only": 0.25450928339078893
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "H2O.ai (paid evaluation, PUBLIC visibility)",
   "axes": {
    "calibration": 89.37957627728649,
    "cost": 56.02061699570665,
    "intelligence": 79.29803124474094,
    "speed": 89.5083860213906
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0321064307460744,
      "ece_score": 93.57871385078512,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 92.33099766610799,
      "score_before_time_limit": 93.57871385078512,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.06538327479433943,
      "ece": 0.05774715480204895,
      "n": 375,
      "score": 87.97883267137907,
      "score_before_time_limit": 88.4505690395902,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.96,
      "interval90_width": 0.15333333333333338,
      "n": 375,
      "nrps": 0.04444906362089437,
      "score": 87.82889849437241,
      "score_before_time_limit": 88.53719606287541,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.0924035075607986
     }
    },
    "score": 89.37957627728649
   },
   "capability": 84.33880376101371,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10147.2130346 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=native). The row's own scored cost basis: 727,319 input + 1,485 output tokens over 1,485 decisions = 490.777104377 tokens/decision. GPU-time cost USD 0.0292401390446342/1,000; old list-price cost USD 0.0444199393939394/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0292401390446342/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.11541544000000001
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02924013904463419
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "H2O-Lightning-31B v1.0.7 (H2O.ai, Gemma-4-31B-it fine-tune, vLLM + h2oai overlay, one-engine FP8 profile, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "paid evaluation (H2O.ai). Evaluator-owned rented Verda GPU VM (1RTXPRO6000.30V, FIN-03, Finland, EU, pay-as-you-go, USD 2.535/h), 1x NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB). Run image built on the digest-pinned official vllm/vllm-openai:v0.30.0 with the author's overlay and shim installed before any test item existed, all 26 weight files sha256-verified at revision ca6e6f6e, then run in an offline container (network none), the cut proven from inside before the sealed input and again before the scored run. Driven serially, one request at a time, by the unchanged self-hosted runner. Measured 22 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; raw latency p50 0.065 s, p95 0.126 s, maximum 39.08 s. The standard self-hosted x2 + 0.15 s adjustment applies (assumption, not measured).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 79.9765674204135,
    "I_sealed": 78.61949506906838,
    "base": 79.29803124474094,
    "excess": -8.482454419010587,
    "gap": 1.3570723513451242,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 85.05319626115181,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 79.2387543252595,
       "judge": 84.17930125247199,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 85.29951690821257,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 70,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 69.57698909187614,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.94393070994124,
       "hard": 80.8292257979457,
       "judge": 68.24978094093717,
       "standard": 52.37985709711295
      }
     },
     "sealed|choice": {
      "cc": 86.12500463167756,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 82.82641378512778,
       "judge": 86.77766895200784,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 81.43542715647708,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 74.54545454545453,
       "judge": 84.61538461538463,
       "standard": 88.05970149253733
      }
     },
     "sealed|score": {
      "cc": 68.2980534190505,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 86.2967685525146,
       "hard": 66.72128429528557,
       "judge": 62.76666308088821,
       "standard": 70.74931960709175
      }
     }
    }
   },
   "jevbench_score": 75.72659612193334,
   "key": "h2o-lightning-31b",
   "last_measured_on": "2026-10-22",
   "licence": "H2O Model Non-Commercial Licence v1.1 (weights; gated on Hugging Face; benchmarking and publishing results are allowed by the licence). Base Gemma 4 under its own licence.",
   "listing": "ranked",
   "measured_in": "v1.6.39",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "h2oai/h2o-lightning-31b@ca6e6f6e8e08e2d21c8c9f45261e05444626647a",
   "noul_decisive": {
    "acc_among_decisive": 0.9195710455764075,
    "decisive": 373,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.18882631775024,
     "cost": 50.57266115000792,
     "intelligence": 80.15012041049178,
     "speed": 89.8424830803211
    },
    "board_release": "v1.6.43",
    "capability": 85.16947336412102,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 2,
    "capability_rank": 3,
    "composite_ci95": {
     "A": [
      72.2022562910307,
      74.22532314173431
     ],
     "B": [
      73.01307775885572,
      75.86853392037305
     ],
     "C": [
      72.2022562910307,
      74.22532314173431
     ]
    },
    "jevbench_score": 73.43686411111796,
    "measured_in": "v1.6.39",
    "method": "v1.6",
    "open_board_rank": 3,
    "rank": 12,
    "ranks": {
     "A": 12,
     "B": 6,
     "C": 12,
     "capability": 8
    },
    "scores": {
     "A": 73.43686411111796,
     "B": 74.68801579994582,
     "C": 73.43686411111796
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.27608948999968563,
     "p50_s_raw": 0.0630447449998428,
     "p95_s_adjusted": 0.37557944640020646,
     "p95_s_raw": 0.11278972320010325
    },
    "views": {
     "Balanced 33:33:33": 69.15516340580727,
     "Emphasis on Accuracy 60:20:20": 73.17014211031464,
     "Emphasis on Cost 20:20:60": 60.293435536349946,
     "Emphasis on Speed 20:60:20": 76.1708694366029,
     "Intelligence only": 80.15012041049178
    }
   },
   "rank": 13,
   "ranked": true,
   "ranks": {
    "A": 13,
    "B": 4,
    "C": 12,
    "capability": 12
   },
   "repo": "https://huggingface.co/h2oai/h2o-lightning-31b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 75.72659612193334,
    "B": 76.4149130958937,
    "C": 75.72659612193334
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27901971299970685,
    "p50_s_raw": 0.06450985649985341,
    "p95_s_adjusted": 0.40135299689954984,
    "p95_s_raw": 0.12567649844977494
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The author's documented JevBench configuration, unmodified: run_default.sh, the one-engine profile (author's README, 'how to run for JevBench'), on vLLM 0.30.0 (official vllm/vllm-openai:v0.30.0 image, digest-pinned) with the author's h2oai/vllm overlay at commit 6bc6d5a8 (18 patched files, sha256-checked against the pin) and the author's shim lightning_31b_shim.py; torch 2.13.0, transformers 5.20.0, torchcodec 0.17.0 hash-locked. Engine: one vLLM engine, FP8 weights (vLLM online FP8 of the bf16 checkpoint, --quantization fp8), two-way reading by default (one-way, read whole, for a record over the 8,192-token two-way window or a request over the 4,096-token latency cap); the profile's own temperatures and yes/no floor 0.801 as shipped. Static review of every author file found no network, subprocess, eval, pickle or trust_remote_code use and no branch on item text. MEASURED AS SHIPPED: the author's own documented JevBench configuration (run_default.sh, one-engine profile): FP8 weights and the two-way reading that this profile applies by default. A different serving choice (for example the author's stock-vLLM one-way fallback) would give different numbers. ALL 1,500 ITEMS ANSWERED IN FULL: none was refused, cut or truncated; the 15 longest items (76,497 to 81,692 input tokens, below the engine's 126,976-token cap) were read whole one-way. 15 decisions took more than 10 seconds (the 15 longest items, maximum 39.08 s); v1.6 applies no 10-second rule. PUBLIC VS SEALED, author statement: the author's README reports 214 of 231 on the public JevBench items from its own client (easy 48/48, original 72/72, hard 94/111). That figure is the author's, on the public items; this row's numbers are what we measured on the sealed draw, and the scorer's standard public-vs-sealed gap rule applies as for every row. TRAINING DATA: the author's training-data statement was not checked. Customer statements about the model, its training and its serving are not independently verified. Measured accuracy on the 15 longest items (answered in full): 7 of 15 correct (46.7 %), versus 1313 of 1485 (88.4 %) on the remaining items; a Score item is correct within half a label step.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 69.31035450645739,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 7.931063519372161
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 69.31035450645739
      },
      "calibration_gen": {
       "accuracy": 0.9702970297029703,
       "cc": 92.23797710576798,
       "cc_by_type": {
        "choice": 96.8186638388123,
        "noul": 92,
        "score": 87.89526747849166
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 92.23797710576798
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 89.44016777438448,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 100,
        "score": 74.82456836380385
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 89.44016777438448
      },
      "fairness": {
       "accuracy": 0.9791666666666666,
       "cc": 94.31870784276252,
       "cc_by_type": {
        "choice": 100,
        "noul": 86.66666666666667,
        "score": 96.28945686162085
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 94.31870784276252
      },
      "jaggedness": {
       "accuracy": 0.8277777777777777,
       "cc": 72.03555111247822,
       "cc_by_type": {
        "choice": 76.90086621751685,
        "noul": 67.56756756756756,
        "score": 71.63821955235024
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 72.03555111247822
      },
      "llm_router": {
       "accuracy": 0.9041533546325878,
       "cc": 82.4432474217307,
       "cc_by_type": {
        "choice": 89.69798362243552,
        "noul": 97.93814432989691,
        "score": 59.69361431285969
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 82.4432474217307
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 57.01060158493846,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": 100,
        "score": 32.887474857908195
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 57.01060158493846
      },
      "multilingual": {
       "accuracy": 0.8472727272727273,
       "cc": 74.4939896452101,
       "cc_by_type": {
        "choice": 84.3495272253016,
        "noul": 69.44444444444444,
        "score": 69.68799726588428
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 74.4939896452101
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 79.72153832282477,
       "cc_by_type": {
        "choice": 100,
        "noul": 73.33333333333334,
        "score": 65.83128163514097
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 79.72153832282477
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.953125,
       "cc": 92.80164393999273,
       "cc_by_type": {
        "choice": 93.993993993994,
        "noul": 100,
        "score": 84.41093782598415
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 92.80164393999273
      },
      "usecase": {
       "accuracy": 0.8227272727272728,
       "cc": 75.452467331574,
       "cc_by_type": {
        "choice": 80.39215686274511,
        "noul": 75,
        "score": 70.96524513197689
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 75.452467331574
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 27.695577686653905,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": -9.770409797181134
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 27.695577686653905
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 86.6601798445292,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 80.7178667225277
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 86.6601798445292
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 90.54110541685141,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 100,
        "score": 79.58791802046574
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 90.54110541685141
      },
      "el": {
       "accuracy": 0.8,
       "cc": 76.74015887855013,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 97.07336304160941
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 76.74015887855013
      },
      "en": {
       "accuracy": 0.8853171155516942,
       "cc": 81.4003355140322,
       "cc_by_type": {
        "choice": 87.25912546832475,
        "noul": 84.89208633093526,
        "score": 72.0497947428366
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 81.4003355140322
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 65.82902920546668,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 7.27501583825978
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 65.82902920546668
      },
      "fi": {
       "accuracy": 1,
       "cc": 99.04348580348609,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.1304574104583
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 99.04348580348609
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 84.01638557859114,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 94.40209791224403
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.01638557859114
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 74.74309771694028,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 100,
        "score": 57.74884622344651
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 74.74309771694028
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 78.05757894622408,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 34.17273683867226
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 78.05757894622408
      },
      "ja": {
       "accuracy": 0.8,
       "cc": 65.43997328394065,
       "cc_by_type": {
        "choice": 100,
        "noul": 80,
        "score": 16.31991985182193
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 65.43997328394065
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 61.36020671382338,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 84.08062014147015
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 61.36020671382338
      },
      "mixed": {
       "accuracy": 1,
       "cc": 98.99576194094993,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.98728582284978
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 98.99576194094993
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 36.274143337969065,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 42.15576334724053
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 36.274143337969065
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8974358974358975,
       "cc": 83.23634974536745,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80,
        "score": 77.32648469949635
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 83.23634974536745
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 74.59524897099772,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 55.606607254772754
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 74.59524897099772
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 58.27807685955187,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 65.89806036588965
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 58.27807685955187
      },
      "tr": {
       "accuracy": 1,
       "cc": 97.24435340666003,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 91.73306021998008
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 97.24435340666003
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 74.51178077969244,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 55.79340685520635
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 74.51178077969244
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 49.78741886942854,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 16.0289232749523
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 49.78741886942854
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.892,
       "cc": 86.05024971775197,
       "cc_by_type": {
        "choice": 86.05024971775197
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 86.05024971775197
      },
      "noul": {
       "accuracy": 0.9146666666666666,
       "cc": 82.93333333333332,
       "cc_by_type": {
        "noul": 82.93333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 82.93333333333332
      },
      "score": {
       "accuracy": 0.8026666666666666,
       "cc": 69.84443725780446,
       "cc_by_type": {
        "score": 69.84443725780446
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 69.84443725780446
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6818181818181818,
       "cc": 51.71841212479776,
       "cc_by_type": {
        "choice": 68,
        "noul": 50,
        "score": 37.155236374393255
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 51.71841212479776
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 80.19260915189135,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 100,
        "score": 77.84490820101566
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 80.19260915189135
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 62.18230459928089,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100,
        "score": 35.32740160272074
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 62.18230459928089
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 35.481357551990506,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 33.33333333333333,
        "score": 31.590271486380892
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 35.481357551990506
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 60.346106569192706,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 76.69049362062161
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 60.346106569192706
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 27.905726525665614,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 66.66666666666667,
        "score": -57.79225438526731
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 27.905726525665614
      },
      "financial_crime": {
       "accuracy": 0.8333333333333334,
       "cc": 82.01582185942733,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 46.047465578282
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 82.01582185942733
      },
      "gaming": {
       "accuracy": 0.8181818181818182,
       "cc": 67.86827777219527,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50,
        "score": 64.05779498905969
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 67.86827777219527
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 61.15211571169912,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 85.95634713509736
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 61.15211571169912
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 76.09635878356498,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100,
        "score": 53.98500782821096
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 76.09635878356498
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 75.32488161969717,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 59.824774058057905
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 75.32488161969717
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 35.17291303717321,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 36.70153481044436
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 35.17291303717321
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 73.65620369970586,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 72.4849258783882
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 73.65620369970586
      },
      "model_routing": {
       "accuracy": 0.8847926267281107,
       "cc": 79.42571605349265,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 100,
        "score": 47.62779751112731
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.42571605349265
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 17.821719144102786,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 88.01410177012026
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 17.821719144102786
      },
      "recruiting": {
       "accuracy": 0.9444444444444444,
       "cc": 74.39779570718791,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 89.86005378823037
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 74.39779570718791
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 70.90997418649607,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 100,
        "score": 27.508247682640942
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 70.90997418649607
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 36.42130234970749,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 52.742167918687684
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 36.42130234970749
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.82840558406987,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.48521675220962
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.82840558406987
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.77948599085776,
      91.12581540138535
     ],
     "I": [
      75.46574808547649,
      82.6119466329837
     ],
     "capability": [
      81.10066647680395,
      86.42295095099455
     ],
     "composite": null,
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.37957627728649,
     "capability": 84.33880376101371,
     "intelligence": 79.29803124474094
    },
    "per_type": {
     "choice": {
      "accuracy": 0.892,
      "calibration": 92.33099766610799,
      "cc": 86.05024971775197,
      "cc_by_type": {
       "choice": 86.05024971775197
      },
      "cc_tiered": 85.58910044641468,
      "cc_tiered_open": 85.05319626115181,
      "cc_tiered_sealed": 86.12500463167756,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 86.05024971775197
     },
     "noul": {
      "accuracy": 0.9146666666666666,
      "calibration": 87.97883267137907,
      "cc": 82.93333333333332,
      "cc_by_type": {
       "noul": 82.93333333333332
      },
      "cc_tiered": 83.36747203234482,
      "cc_tiered_open": 85.29951690821257,
      "cc_tiered_sealed": 81.43542715647708,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 82.93333333333332
     },
     "score": {
      "accuracy": 0.8026666666666666,
      "calibration": 87.82889849437241,
      "cc": 69.84443725780446,
      "cc_by_type": {
       "score": 69.84443725780446
      },
      "cc_tiered": 68.93752125546332,
      "cc_tiered_open": 69.57698909187614,
      "cc_tiered_sealed": 68.2980534190505,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 69.84443725780446
     }
    },
    "raw_on_A": {
     "calibration": 85.46585036403003,
     "capability": 81.17415269279076,
     "intelligence": 76.88245502155146
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 72.05759635681919,
    "Emphasis on Accuracy 60:20:20": 74.78909191522838,
    "Emphasis on Cost 20:20:60": 64.65419364734008,
    "Emphasis on Speed 20:60:20": 78.15230791562422,
    "Intelligence only": 79.29803124474094
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "H2O.ai",
   "axes": {
    "calibration": 89.95399367673842,
    "cost": 78.58891530847164,
    "intelligence": 60.03839906196252,
    "speed": 92.37207479774074
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03794010455551056,
      "ece_score": 92.4119790888979,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 92.4119790888979,
      "score_before_time_limit": 92.4119790888979,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.1156787859671944,
      "ece": 0.055451181905507255,
      "n": 375,
      "score": 88.90976361889855,
      "score_before_time_limit": 88.90976361889855,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.952,
      "interval90_width": 0.45600000000000007,
      "n": 375,
      "nrps": 0.08555524714180213,
      "score": 88.54023832241884,
      "score_before_time_limit": 88.54023832241884,
      "time_limit_failed_share": 0,
      "top_ece": 0.0718199932049105
     }
    },
    "score": 89.95399367673842
   },
   "capability": 74.99619636935047,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 854,867 input + 0 output tokens over 1,624 decisions = 526.395935961 tokens/decision. GPU-time cost USD 0.00517252767578001/1,000; old list-price cost USD 0.0157918780788177/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00517252767578001/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005172527675780014
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "H2O-Lightning-4B v1.1 (H2O.ai, Qwen3.5-4B fine-tune, stock vLLM + open shim)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-04",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX 5090, official vLLM 0.30.0 image + the author's open standard-library shim (h2o_lightning_shim.py) as published on the model card, offline (all credentials unset), driven over loopback only by the JevBench client; label-free inputs; pod destroyed afterwards; JevBench add-request (GitHub #181), measured 4 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 61.05316077664259,
    "I_sealed": 59.02363734728246,
    "base": 60.03839906196252,
    "excess": -0.5438642130781113,
    "gap": 2.0295234293601325,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.18196275852294,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 69.80182447310474,
       "judge": 80.22412656558997,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 60.94202898550725,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 50,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 47.035490585897584,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.83317199906027,
       "hard": 55.09009420986275,
       "judge": 42.073705022183916,
       "standard": 34.47012097695641
      }
     },
     "sealed|choice": {
      "cc": 64.65667579231118,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 72.06414899120539,
       "hard": 60.80316742081447,
       "judge": 67.41154562383613,
       "standard": 64.52765118857003
      }
     },
     "sealed|noul": {
      "cc": 68.08195642942992,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 55.33980582524271,
       "judge": 68.42105263157893,
       "standard": 83.09859154929578
      }
     },
     "sealed|score": {
      "cc": 44.33227982010628,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.681000195734974,
       "hard": 40.36020718747171,
       "judge": 45.957875906717405,
       "standard": 40.163670767644376
      }
     }
    }
   },
   "jevbench_score": 77.93772981332452,
   "key": "h2o-lightning-4b",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "h2oai/h2o-lightning-4b@193ad740925b176a3b70a5a13a7cff2f2fadd01e",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.848,
    "decisive_rate": 1,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.95399367673842,
     "cost": 64.04698652895101,
     "intelligence": 60.03839906196252,
     "speed": 92.61084474720658
    },
    "board_release": "v1.6.43",
    "capability": 74.99619636935047,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 13,
    "capability_rank": 24,
    "composite_ci95": {
     "A": [
      70.99731037439084,
      75.13409530573585
     ],
     "B": [
      66.92981430724642,
      72.55375740934666
     ],
     "C": [
      58.58968210866478,
      75.13409530573585
     ]
    },
    "jevbench_score": 73.82224524014528,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 2,
    "rank": 10,
    "ranks": {
     "A": 10,
     "B": 18,
     "C": 10,
     "capability": 44
    },
    "scores": {
     "A": 73.82224524014528,
     "B": 70.5813765538927,
     "C": 73.82224524014528
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20856922594830393,
     "p50_s_raw": 0.02928461297415197,
     "p95_s_adjusted": 0.2628241665661335,
     "p95_s_raw": 0.056412083283066745
    },
    "views": {
     "Balanced 33:33:33": 69.65823125473824,
     "Emphasis on Accuracy 60:20:20": 65.4626515419746,
     "Emphasis on Cost 20:20:60": 67.2997420088874,
     "Emphasis on Speed 20:60:20": 77.3237822737329,
     "Intelligence only": 60.038399061962515
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a8 (scorer output sha256 7a5b4d4c7b7c, method option B / O1S)"
   },
   "rank": 4,
   "ranked": true,
   "ranks": {
    "A": 4,
    "B": 21,
    "C": 3,
    "capability": 63
   },
   "repo": "https://huggingface.co/h2oai/h2o-lightning-4b",
   "repriced_in": "v1.6.37",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 77.93772981332452,
    "B": 73.55209245183703,
    "C": 77.93772981332452
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21219652900472283,
    "p50_s_raw": 0.031098264502361417,
    "p95_s_adjusted": 0.2729318660800345,
    "p95_s_raw": 0.06146593304001726
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "supersedes_base_row": "v1.6.1",
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.64761887040858,
      90.96540455486856
     ],
     "I": [
      54.48601679476801,
      63.869097672212845
     ],
     "capability": [
      70.2236559039794,
      76.81358157643643
     ],
     "composite": null,
     "seed": 44
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.95399367673842,
     "capability": 74.99619636935047,
     "intelligence": 60.03839906196252
    },
    "per_type": {
     "choice": {
      "accuracy": 0.744,
      "calibration": 92.4119790888979,
      "cc": 67.26125631139698,
      "cc_by_type": {
       "choice": 67.26125631139698
      },
      "cc_tiered": 69.91931927541705,
      "cc_tiered_open": 75.18196275852294,
      "cc_tiered_sealed": 64.65667579231118,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.26125631139698
     },
     "noul": {
      "accuracy": 0.848,
      "calibration": 88.90976361889855,
      "cc": 69.6,
      "cc_by_type": {
       "noul": 69.6
      },
      "cc_tiered": 64.51199270746858,
      "cc_tiered_open": 60.94202898550725,
      "cc_tiered_sealed": 68.08195642942992,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 69.6
     },
     "score": {
      "accuracy": 0.5893333333333334,
      "calibration": 88.54023832241884,
      "cc": 45.79426376408395,
      "cc_by_type": {
       "score": 45.79426376408395
      },
      "cc_tiered": 45.683885203001935,
      "cc_tiered_open": 47.035490585897584,
      "cc_tiered_sealed": 44.33227982010628,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 45.79426376408395
     }
    },
    "raw_on_A": {
     "calibration": 86.31592326935305,
     "capability": 74.50666727956846,
     "intelligence": 62.69741128978386
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 74.61530127079625,
    "Emphasis on Accuracy 60:20:20": 68.01032892375518,
    "Emphasis on Cost 20:20:60": 76.15553218270043,
    "Emphasis on Speed 20:60:20": 80.83055571908528,
    "Intelligence only": 60.038399061962515
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "H2O.ai",
   "axes": {
    "calibration": 89.65360096890336,
    "cost": 83.2204883200473,
    "intelligence": 64.42926862934559,
    "speed": 93.86173474461954
   },
   "base_model": "Qwen/Qwen3.5-4B",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07385800042536797,
      "ece_score": 85.22839991492641,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 354,
      "n_tvd": 0,
      "score": 85.22839991492641,
      "score_before_time_limit": 85.22839991492641,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.08427481593744356,
      "ece": 0.048296574505722646,
      "n": 375,
      "score": 90.34068509885546,
      "score_before_time_limit": 90.34068509885546,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.944,
      "interval90_width": 0.21555555555555556,
      "n": 375,
      "nrps": 0.05677503680041495,
      "score": 93.39171789292823,
      "score_before_time_limit": 93.39171789292823,
      "time_limit_failed_share": 0,
      "top_ece": 0.037695302670510286
     }
    },
    "score": 89.65360096890336
   },
   "capability": 77.04143479912447,
   "code_revision": "Author's own serving stack from the model repo at cc18ee33 (serve.sh 378e5c05, h2o_lightning_shim.py bc386d26, box.py, assets.py, pooled_readout.py; POST /v1/systemone), README 'For benchmark maintainers: JevBench' default: two-way reading, one engine (TWO_WAY=1, ONE_ENGINE=1), shipped serve_config.json (32,000-token state trim), MAX_LEN 40960, on vLLM 0.30.0 (official image digest 8a69ffad) with the author's h2oai/vllm per-request-attention overlay @b05cc29a (30 files, every sha256 checked against the author's engine/vllm_per_request_attention.SHA256SUMS). Evaluator kit hash 99f91ea7, independent source reviews PASS; no trust_remote_code.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 547,826 input + 1,489 output tokens over 1,489 decisions = 368.915379449 tokens/decision. GPU-time cost USD 0.00362507549899706/1,000; old list-price cost USD 0.0111874613834788/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00362507549899706/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.014370400000000004
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003625075498997057
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of v1.4.1 (paid evaluation, public). FAMILY: new weights (model.safetensors c9b6290d) of a family with an earlier row (H2O-Lightning-4B v1.1, which stays on the board unchanged); an earlier v1.4.0 measurement attempt stopped before completion because of an evaluator-side monitoring error and has no row. DRAW: first row measured on the 27 Oct 2026 regular draw (fresh sealed items; not yet equated to the reference scale, so its numbers are shown as published, offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own draw. CONTEXT: 5 items whose state exceeds the shipped 32,000-token limit are answered on the author's own head/tail state trim (pre-registered positions); cost is priced on the trimmed prompt, the untrimmed count is disclosed. ENGINE: the author's documented JevBench configuration, two-way reading on one engine with the h2oai vLLM overlay and an open request shim (README: the one-way TWO_WAY=0 setting is not the measured configuration). PROVENANCE, author statements (unverified): fine-tune of Qwen/Qwen3.5-4B. HARDWARE: one Verda RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, EU); latency reflects that card.",
   "display": "H2O-Lightning-4B v1.4.1",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-27",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.13",
    "results_sha256": "68b75d4d8b25a0775b8620c1a7b522e66b8abcf3b34549f757eb04cc4769bed7",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA RTX PRO 6000 Blackwell Server Edition (1RTXPRO6000.30V, FIN-03, Finland, EU, pay-as-you-go non-spot, USD 2.649/h all-in), instance id 15b01e61-e76c-4e50-80c0-7aaebf2551d5. Container --network none (cut proven from inside before the sealed input), every credential unset, read-only weights, all 24 model files sha256-verified; driven one request at a time by the frozen JevBench pod driver. The standard self-hosted x2 + 0.15 s latency adjustment applies. Measured 27 Oct 2026 15:22Z on the frozen v1.6-regular-20261027 self-hosted input (sha256 9b2f32ed...); determinism 50/50 bit-identical, 0 argmax flips; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 57.21644687422314,
    "I_sealed": 71.64209038446805,
    "base": 64.42926862934559,
    "excess": -16.999031152683155,
    "gap": -14.42564351024491,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 67.09558133213523,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 62.25228059138095,
       "judge": 68.35860250494397,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 62.69220151828847,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 30.000000000000004,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 41.86155777224573,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.61440648604576,
       "hard": 49.98132468914773,
       "judge": 40.830243927096824,
       "standard": 24.292570349265084
      }
     },
     "sealed|choice": {
      "cc": 72.49208438928004,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 92.13887979037014,
       "hard": 61.96915275723643,
       "judge": 74.11561691113029,
       "standard": 81.27925117004682
      }
     },
     "sealed|noul": {
      "cc": 79.01922663172101,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 88.50574712643677,
       "hard": 70.73170731707317,
       "judge": 82.22222222222221,
       "standard": 86.04651162790698
      }
     },
     "sealed|score": {
      "cc": 63.41496013240311,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 83.32705582731622,
       "hard": 46.85886109180122,
       "judge": 75.11853848834807,
       "standard": 69.01574283223289
      }
     }
    }
   },
   "jevbench_score": 81.06161765725506,
   "key": "h2o-lightning-4b-v1.4.1",
   "last_measured_on": "2026-10-27",
   "licence": "Apache-2.0 (HF metadata and LICENSE), public, not gated",
   "listing": "unranked",
   "measured_in": "v1.7.13",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "h2oai/h2o-lightning-4b@cc18ee330e6895c41b9a60f37a0dd122dc4c3461",
   "new_in": "v1.7.13",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale: its first anchor runs used models weaker than this row, and stronger anchors are being measured. On the 300 public items every draw shares, this version scores 57.2 Intelligence vs 61.1 for H2O-Lightning-4B v1.1. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.8933333333333333,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/h2oai/h2o-lightning-4b",
   "scores": {
    "A": 81.06161765725506,
    "B": 77.08189535244315,
    "C": 81.06161765725506
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18970937999997659,
    "p50_s_raw": 0.019854689999988295,
    "p95_s_adjusted": 0.21663953800001534,
    "p95_s_raw": 0.03331976900000768
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 100
      },
      "calibration_gen": {
       "accuracy": 0.35714285714285715,
       "cc": -4.89156700112579,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": 0,
        "score": -36.752623081299454
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.7368421052631579,
       "cc": 59.53420057018902,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 60.00000000000001,
        "score": 45.26926837723371
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 59.53420057018902
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.25875038803422,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.77625116410262
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 99.25875038803422
      },
      "jaggedness": {
       "accuracy": 0.6363636363636364,
       "cc": 31.88432376484649,
       "cc_by_type": {
        "choice": 87.65432098765433,
        "noul": -33.333333333333336,
        "score": 41.33198364021848
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 31.88432376484649
      },
      "llm_router": {
       "accuracy": 0.7333333333333333,
       "cc": 67.5745138219251,
       "cc_by_type": {
        "choice": 57.696827262044664,
        "noul": 91.30434782608697,
        "score": 53.722366377643624
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 67.5745138219251
      },
      "long_state": {
       "accuracy": 0.6,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 67.74193548387098
      },
      "ml_usecase": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 100
      },
      "multilingual": {
       "accuracy": 0.8121606948968513,
       "cc": 72.725836190313,
       "cc_by_type": {
        "choice": 69.60551843069629,
        "noul": 82.78688524590163,
        "score": 65.7851048943411
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 72.725836190313
      },
      "negation_pair_gen": {
       "accuracy": 0.8636363636363636,
       "cc": 75,
       "cc_by_type": {
        "choice": 100,
        "noul": 50
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 75
      },
      "option_order": {
       "accuracy": 0.8888888888888888,
       "cc": 85.71428571428571,
       "cc_by_type": {
        "choice": 85.71428571428571
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 85.71428571428571
      },
      "perturb_pair": {
       "accuracy": 0.9285714285714286,
       "cc": 83.80919215666876,
       "cc_by_type": {
        "choice": 100,
        "noul": 80,
        "score": 71.42757647000629
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 83.80919215666876
      },
      "usecase": {
       "accuracy": 0.796923076923077,
       "cc": 69.10836257325546,
       "cc_by_type": {
        "choice": 72.2830051057622,
        "noul": 69.23076923076923,
        "score": 65.81131338323493
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 69.10836257325546
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8571428571428571,
       "cc": 87.01375178729741,
       "cc_by_type": {
        "choice": 72.6027397260274,
        "noul": 100,
        "score": 88.43851563586483
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 87.01375178729741
      },
      "cs": {
       "accuracy": 0.7837837837837838,
       "cc": 63.87009281292052,
       "cc_by_type": {
        "choice": 67.91443850267379,
        "noul": 55.55555555555556,
        "score": 68.14028438053221
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 63.87009281292052
      },
      "da": {
       "accuracy": 0.75,
       "cc": 45.19457663032557,
       "cc_by_type": {
        "choice": 62.73291925465838,
        "noul": 63.63636363636365,
        "score": 9.214446999954696
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 45.19457663032557
      },
      "de": {
       "accuracy": 0.75,
       "cc": 59.28318187641174,
       "cc_by_type": {
        "choice": 72.21507589400566,
        "noul": 50,
        "score": 55.634469735229565
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 59.28318187641174
      },
      "el": {
       "accuracy": 0.6785714285714286,
       "cc": 52.48989676074584,
       "cc_by_type": {
        "choice": 42.307692307692314,
        "noul": 39.99999999999999,
        "score": 75.16199797454522
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 52.48989676074584
      },
      "en": {
       "accuracy": 0.790990990990991,
       "cc": 67.77719607032161,
       "cc_by_type": {
        "choice": 73.7872421695951,
        "noul": 70.49180327868851,
        "score": 59.052542762681234
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 67.77719607032161
      },
      "es": {
       "accuracy": 0.8378378378378378,
       "cc": 74.86761082279651,
       "cc_by_type": {
        "choice": 78.34803528468322,
        "noul": 77.77777777777777,
        "score": 68.47701940592852
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 74.86761082279651
      },
      "fi": {
       "accuracy": 0.8269230769230769,
       "cc": 75.42269476291459,
       "cc_by_type": {
        "choice": 58.85714285714286,
        "noul": 89.4736842105263,
        "score": 77.93725722107457
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 75.42269476291459
      },
      "fr": {
       "accuracy": 0.84375,
       "cc": 80.84455880834453,
       "cc_by_type": {
        "choice": 69.38775510204081,
        "noul": 100,
        "score": 73.14592132299278
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 80.84455880834453
      },
      "hi": {
       "accuracy": 0.6486486486486487,
       "cc": 59.80967331919211,
       "cc_by_type": {
        "choice": 58.677685950413235,
        "noul": 77.77777777777777,
        "score": 42.9735562293853
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 59.80967331919211
      },
      "id": {
       "accuracy": 0.9024390243902439,
       "cc": 85.91111455247433,
       "cc_by_type": {
        "choice": 81.5668202764977,
        "noul": 100,
        "score": 76.16652338092528
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 85.91111455247433
      },
      "it": {
       "accuracy": 0.7777777777777778,
       "cc": 64.10095109593591,
       "cc_by_type": {
        "choice": 76.04259094942324,
        "noul": 83.33333333333333,
        "score": 32.926929005051186
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 64.10095109593591
      },
      "ja": {
       "accuracy": 0.8636363636363636,
       "cc": 78.75360634477363,
       "cc_by_type": {
        "choice": 68.91191709844558,
        "noul": 100,
        "score": 67.34890193587529
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 78.75360634477363
      },
      "ko": {
       "accuracy": 0.7674418604651163,
       "cc": 69.15139945081272,
       "cc_by_type": {
        "choice": 55.88235294117647,
        "noul": 80,
        "score": 71.57184541126168
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 69.15139945081272
      },
      "mixed": {
       "accuracy": 0.9375,
       "cc": 86.06607931725432,
       "cc_by_type": {
        "choice": 94.5994599459946,
        "noul": 89.99999999999999,
        "score": 73.59877800576837
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 86.06607931725432
      },
      "nl": {
       "accuracy": 0.8611111111111112,
       "cc": 67.59191440851161,
       "cc_by_type": {
        "choice": 85.84905660377359,
        "noul": 80,
        "score": 36.92668662176126
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 67.59191440851161
      },
      "no": {
       "accuracy": 0.8235294117647058,
       "cc": 71.48247682213808,
       "cc_by_type": {
        "choice": 69.01893287435456,
        "noul": 86.66666666666667,
        "score": 58.76183092539302
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 71.48247682213808
      },
      "pl": {
       "accuracy": 0.7894736842105263,
       "cc": 67.83842757895876,
       "cc_by_type": {
        "choice": 66.47152740819585,
        "noul": 76.47058823529412,
        "score": 60.573167093386274
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 67.83842757895876
      },
      "pt": {
       "accuracy": 0.8085106382978723,
       "cc": 75.1508004595795,
       "cc_by_type": {
        "choice": 64.05990016638934,
        "noul": 83.33333333333333,
        "score": 78.05916787901586
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 75.1508004595795
      },
      "sv": {
       "accuracy": 0.782608695652174,
       "cc": 69.2519737584483,
       "cc_by_type": {
        "choice": 53.07262569832402,
        "noul": 83.33333333333333,
        "score": 71.34996224368757
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 69.2519737584483
      },
      "tr": {
       "accuracy": 0.8260869565217391,
       "cc": 76.21533995484549,
       "cc_by_type": {
        "choice": 65.79247434435577,
        "noul": 80,
        "score": 82.85354552018069
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 76.21533995484549
      },
      "uk": {
       "accuracy": 0.7857142857142857,
       "cc": 69.60134146941199,
       "cc_by_type": {
        "choice": 72.6027397260274,
        "noul": 100,
        "score": 36.20128468220858
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 69.60134146941199
      },
      "zh": {
       "accuracy": 0.8536585365853658,
       "cc": 81.3528808789535,
       "cc_by_type": {
        "choice": 75.20661157024794,
        "noul": 100,
        "score": 68.85203106661254
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 81.3528808789535
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7893333333333333,
       "cc": 71.26430974729968,
       "cc_by_type": {
        "choice": 71.26430974729968
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 71.26430974729968
      },
      "noul": {
       "accuracy": 0.8933333333333333,
       "cc": 78.66666666666666,
       "cc_by_type": {
        "noul": 78.66666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 78.66666666666666
      },
      "score": {
       "accuracy": 0.7413333333333333,
       "cc": 63.284787325054424,
       "cc_by_type": {
        "score": 63.284787325054424
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 63.284787325054424
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7916666666666666,
       "cc": 80.45755248342309,
       "cc_by_type": {
        "choice": 64.41808747220162,
        "noul": 100,
        "score": 76.95456997806764
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 80.45755248342309
      },
      "code_linting": {
       "accuracy": 0.782608695652174,
       "cc": 63.05895346166627,
       "cc_by_type": {
        "choice": 81.62890385793018,
        "noul": 83.33333333333333,
        "score": 24.214623193735296
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 63.05895346166627
      },
      "customer_support": {
       "accuracy": 0.8636363636363636,
       "cc": 78.73554240052313,
       "cc_by_type": {
        "choice": 77.85977859778598,
        "noul": 73.33333333333334,
        "score": 85.01351527045007
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 78.73554240052313
      },
      "demand_forecasting": {
       "accuracy": 0.5757575757575758,
       "cc": 24.577146058628646,
       "cc_by_type": {
        "choice": 54.510993176648974,
        "noul": -7.692307692307687,
        "score": 26.91275269154466
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 24.577146058628646
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7741935483870968,
       "cc": 67.52628994392593,
       "cc_by_type": {
        "choice": 66.22097114707952,
        "noul": 76.47058823529412,
        "score": 59.88731044940418
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 67.52628994392593
      },
      "feature_extraction": {
       "accuracy": 0.8333333333333334,
       "cc": 76.18253812798172,
       "cc_by_type": {
        "choice": 69.84924623115577,
        "noul": 85.71428571428572,
        "score": 72.98408243850363
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 76.18253812798172
      },
      "financial_crime": {
       "accuracy": 0.7936507936507936,
       "cc": 64.1475845766504,
       "cc_by_type": {
        "choice": 74.48165869218504,
        "noul": 63.63636363636365,
        "score": 54.324731401402495
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 64.1475845766504
      },
      "gaming": {
       "accuracy": 0.7413793103448276,
       "cc": 72.40762729797409,
       "cc_by_type": {
        "choice": 56.97211155378488,
        "noul": 100,
        "score": 60.25077034013737
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 72.40762729797409
      },
      "insurance_claims": {
       "accuracy": 0.6935483870967742,
       "cc": 55.53594918238844,
       "cc_by_type": {
        "choice": 46.54788418708242,
        "noul": 64.70588235294117,
        "score": 55.354081007141744
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 55.53594918238844
      },
      "knowledge_graphs": {
       "accuracy": 0.7611940298507462,
       "cc": 59.29998764919901,
       "cc_by_type": {
        "choice": 73.01349325337331,
        "noul": 69.23076923076923,
        "score": 35.65570046345448
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 59.29998764919901
      },
      "lead_generation": {
       "accuracy": 0.8666666666666667,
       "cc": 79.56362072113461,
       "cc_by_type": {
        "choice": 66.14950634696756,
        "noul": 88.57142857142857,
        "score": 83.9699272450077
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 79.56362072113461
      },
      "legal_compliance": {
       "accuracy": 0.8571428571428571,
       "cc": 81.26168767506469,
       "cc_by_type": {
        "choice": 81.94583751253761,
        "noul": 85.71428571428572,
        "score": 76.12493979837073
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 81.26168767506469
      },
      "llm_guardrails": {
       "accuracy": 0.9436619718309859,
       "cc": 89.12328421131313,
       "cc_by_type": {
        "choice": 91.85336048879836,
        "noul": 100,
        "score": 75.51649214514102
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 89.12328421131313
      },
      "model_routing": {
       "accuracy": 0.7905405405405406,
       "cc": 71.1788588124594,
       "cc_by_type": {
        "choice": 73.51566557036526,
        "noul": 88.23529411764706,
        "score": 51.78561674936586
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 71.1788588124594
      },
      "moderation_trust_safety": {
       "accuracy": 0.8305084745762712,
       "cc": 74.50994519671575,
       "cc_by_type": {
        "choice": 73.8562091503268,
        "noul": 76.47058823529412,
        "score": 73.20303820452635
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 74.50994519671575
      },
      "other": {
       "accuracy": 0.84375,
       "cc": 83.47574417398887,
       "cc_by_type": {
        "choice": 66.70366259711432,
        "noul": 100,
        "score": 83.72356992485231
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 83.47574417398887
      },
      "recruiting": {
       "accuracy": 0.8307692307692308,
       "cc": 72.90339667135217,
       "cc_by_type": {
        "choice": 65.39293439077144,
        "noul": 92,
        "score": 61.317255623285064
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 72.90339667135217
      },
      "risk_assessment": {
       "accuracy": 0.8125,
       "cc": 72.53718783602082,
       "cc_by_type": {
        "choice": 56.06694560669455,
        "noul": 100,
        "score": 61.54461790136791
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 72.53718783602082
      },
      "scientific_discovery": {
       "accuracy": 0.8095238095238095,
       "cc": 61.74363544202664,
       "cc_by_type": {
        "choice": 70.55214723926382,
        "noul": 39.99999999999999,
        "score": 74.6787590868161
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 61.74363544202664
      },
      "search_retrieval": {
       "accuracy": 0.9552238805970149,
       "cc": 92.09406677749814,
       "cc_by_type": {
        "choice": 100,
        "noul": 85.71428571428572,
        "score": 90.56791461820873
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 92.09406677749814
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.88959779899693,
      91.30823428308344
     ],
     "I": [
      58.81493806637485,
      68.61698271399555
     ],
     "capability": [
      72.63899066827305,
      79.2191043845938
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.65360096890336,
     "capability": 77.04143479912447,
     "intelligence": 64.42926862934559
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7893333333333333,
      "calibration": 85.22839991492641,
      "cc": 71.26430974729968,
      "cc_by_type": {
       "choice": 71.26430974729968
      },
      "cc_tiered": 69.79383286070762,
      "cc_tiered_open": 67.09558133213523,
      "cc_tiered_sealed": 72.49208438928004,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 71.26430974729968
     },
     "noul": {
      "accuracy": 0.8933333333333333,
      "calibration": 90.34068509885546,
      "cc": 78.66666666666666,
      "cc_by_type": {
       "noul": 78.66666666666666
      },
      "cc_tiered": 70.85571407500474,
      "cc_tiered_open": 62.69220151828847,
      "cc_tiered_sealed": 79.01922663172101,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 78.66666666666666
     },
     "score": {
      "accuracy": 0.7413333333333333,
      "calibration": 93.39171789292823,
      "cc": 63.284787325054424,
      "cc_by_type": {
       "score": 63.284787325054424
      },
      "cc_tiered": 52.63825895232442,
      "cc_tiered_open": 41.86155777224573,
      "cc_tiered_sealed": 63.41496013240311,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 63.284787325054424
     }
    },
    "raw_on_A": {
     "calibration": 84.5902892636588,
     "capability": 74.896854292599,
     "intelligence": 65.2034193215392
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 78.55225691324722,
    "Emphasis on Accuracy 60:20:20": 72.21996855165494,
    "Emphasis on Cost 20:20:60": 80.35525977937333,
    "Emphasis on Speed 20:60:20": 84.03492033711242,
    "Intelligence only": 64.42926862934559
   }
  },
  {
   "adapter_id": "images/hopper/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "HopitAI",
   "axes": {
    "calibration": 88.80153061371698,
    "cost": 76.79820577848395,
    "intelligence": 33.59530941839744,
    "speed": 90.6039582415622
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.054931646080717024,
      "ece_score": 89.0136707838566,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 87.5894520513149,
      "score_before_time_limit": 89.0136707838566,
      "time_limit_failed_share": 0.016
     },
     "noul": {
      "abstention_rate": 0.43733333333333335,
      "brier": 0.1320524837128236,
      "ece": 0.03940530319869939,
      "n": 375,
      "score": 91.13633734041736,
      "score_before_time_limit": 92.11893936026013,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.9733333333333334,
      "interval90_width": 0.6148888888888887,
      "n": 375,
      "nrps": 0.110200461495695,
      "score": 87.67880244941868,
      "score_before_time_limit": 89.10447403396208,
      "time_limit_failed_share": 0.016,
      "top_ece": 0.053855028912531695
     }
    },
    "score": 88.80153061371698
   },
   "capability": 61.19842001605721,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 980,819 input + 0 output tokens over 1,624 decisions = 603.952586207 tokens/decision. GPU-time cost USD 0.00593462307286499/1,000; old list-price cost USD 0.0181185775862069/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00593462307286499/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005934623072864991
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 22,
    "n_over_limit_answered_ok": 22,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Hopper",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 40.742066525947536,
    "I_sealed": 27.963579853032826,
    "base": 34.35282318949018,
    "excess": 10.205099030476466,
    "gap": 12.77848667291471,
    "penalty": 0.9779490096952353,
    "per_type_split": {
     "open|choice": {
      "cc": 68.23228363694746,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 62.25228059138095,
       "judge": 72.31377719182596,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 18.554865424430645,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 30.434782608695656,
       "judge": -19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 35.439050516464505,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.1505221295817,
       "hard": 30.729313281528604,
       "judge": 33.331159411757824,
       "standard": 37.66462583683774
      }
     },
     "sealed|choice": {
      "cc": 56.856747897970024,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 68.07331313280615,
       "hard": 48.585972850678715,
       "judge": 59.031657355679705,
       "standard": 64.52765118857003
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 56.00000000000001,
       "hard": -22.330097087378643,
       "judge": 0,
       "standard": -1.4084507042253502
      }
     },
     "sealed|score": {
      "cc": 27.033991661128457,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 62.587460120023806,
       "hard": 21.43215746572834,
       "judge": 26.893973849966592,
       "standard": 20.67095253922381
      }
     }
    }
   },
   "jevbench_score": 27.745611075479957,
   "key": "hopper",
   "last_measured_on": "2026-10-02",
   "licence": "Component-specific terms recorded in RESULT.md; submitted adapter release and Qwen base retain their respective terms",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9428571428571428,
    "decisive_rate": 0.5660377358490566,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.07902805935959,
     "cost": 62.25627699896333,
     "intelligence": 34.73970870443837,
     "speed": 91.11434937226889
    },
    "board_release": "v1.6.43",
    "capability": 62.40936838189898,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 50,
    "capability_rank": 75,
    "composite_ci95": {
     "A": [
      19.749752505423817,
      34.23513789747224
     ],
     "B": [
      16.849888107534454,
      30.36407576551454
     ],
     "C": [
      13.715105906544313,
      23.774401317689065
     ]
    },
    "jevbench_score": 28.852652318563656,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 72,
    "rank": 104,
    "ranks": {
     "A": 104,
     "B": 105,
     "C": 114,
     "capability": 118
    },
    "scores": {
     "A": 28.852652318563656,
     "B": 25.218775497922103,
     "C": 20.036564110113652
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26295550270006063,
     "p50_s_raw": 0.0564777513500303,
     "p95_s_adjusted": 0.2942271867766976,
     "p95_s_raw": 0.07211359338834882
    },
    "views": {
     "Balanced 33:33:33": 25.942845868326373,
     "Emphasis on Accuracy 60:20:20": 21.285843260250505,
     "Emphasis on Cost 20:20:60": 27.444368192058636,
     "Emphasis on Speed 20:60:20": 31.034805184083435,
     "Intelligence only": 16.77021030692692
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 170,
   "ranked": true,
   "ranks": {
    "A": 170,
    "B": 172,
    "C": 173,
    "capability": 160
   },
   "repo": "https://huggingface.co/HopitAI/hopper",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 27.745611075479957,
    "B": 23.79817626105929,
    "C": 19.267785469083307
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2637434290722013,
    "p50_s_raw": 0.056871714536100626,
    "p95_s_adjusted": 0.3299305472988635,
    "p95_s_raw": 0.08996527364943174
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-4B plus HopitAI/hopper LoRA",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.30075058183441,
      88.36926572708367
     ],
     "I": [
      28.90778639701942,
      36.25273459922729
     ],
     "capability": [
      56.5453532049558,
      61.563676847829825
     ],
     "composite": null,
     "seed": 41
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.80153061371698,
     "capability": 61.19842001605721,
     "intelligence": 33.59530941839744
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6826666666666666,
      "calibration": 87.5894520513149,
      "cc": 59.41759896933583,
      "cc_by_type": {
       "choice": 59.41759896933583
      },
      "cc_tiered": 62.54451576745874,
      "cc_tiered_open": 68.23228363694746,
      "cc_tiered_sealed": 56.856747897970024,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.41759896933583
     },
     "noul": {
      "accuracy": 0.528,
      "calibration": 91.13633734041736,
      "cc": 5.600000000000005,
      "cc_by_type": {
       "noul": 5.600000000000005
      },
      "cc_tiered": 9.277432712215322,
      "cc_tiered_open": 18.554865424430645,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.600000000000005
     },
     "score": {
      "accuracy": 0.496,
      "calibration": 87.67880244941868,
      "cc": 30.07705243305412,
      "cc_by_type": {
       "score": 30.07705243305412
      },
      "cc_tiered": 31.23652108879648,
      "cc_tiered_open": 35.439050516464505,
      "cc_tiered_sealed": 27.033991661128457,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.07705243305412
     }
    },
    "raw_on_A": {
     "calibration": 86.24628252354488,
     "capability": 63.31122802105749,
     "intelligence": 40.376173518570106
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.016000000000000014,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 25.162895027625723,
    "Emphasis on Accuracy 60:20:20": 19.91321496320858,
    "Emphasis on Cost 20:20:60": 28.26327831461998,
    "Emphasis on Speed 20:60:20": 29.740964202669424,
    "Intelligence only": 15.16686871225422
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "HopitAI (@ArchieIndian)",
   "axes": {
    "calibration": 84.63452107883388,
    "cost": 66.55744455639032,
    "intelligence": 51.6576814732076,
    "speed": 83.63977704006717
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11381338060621848,
      "ece_score": 77.2373238787563,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 77.2373238787563,
      "score_before_time_limit": 77.2373238787563,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2075471698113207,
      "brier": 0.09205588963446744,
      "ece": 0.056376852598614385,
      "n": 371,
      "score": 88.72462948027713,
      "score_before_time_limit": 88.72462948027713,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.964769647696477,
      "interval90_width": 0.40492321589882563,
      "n": 369,
      "nrps": 0.08202603571136496,
      "score": 87.94160987746824,
      "score_before_time_limit": 87.94160987746824,
      "time_limit_failed_share": 0,
      "top_ece": 0.0795708833696351
     }
    },
    "score": 84.63452107883388
   },
   "capability": 68.14610127602074,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 854,727 input + 0 output tokens over 1,477 decisions = 578.69126608 tokens/decision. GPU-time cost USD 0.0130242237361284/1,000; old list-price cost USD 0.0289345633039946/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0130242237361284/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.028490900000000006
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01302422373612839
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Hopper 12B trained (gemma-4-12B-it frozen + LoRA r32 unmerged, one forward pass, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0/v1.6.1 pool. Serving code github.com/hopit-ai/hopper tag 12b-1.1.1, installed with the author's own one-line pip install; adapter HopitAI/hopper@4b605f25eb903c2cf4d121040ffb097240b3bc37 (branch hopper-12b), LoRA r32 alpha 64, loaded UNMERGED because the author states a bf16 merge changes answers; base google/gemma-4-12B-it@707f0a3b, frozen; readout map maps/hopper-12b-trained-readout.json as shipped in their package. 1.1.1 is their own fix for corrupted option scores at prompt lengths one above a multiple of 32. Their flags left at their defaults: --shortlist tournament (their measured default) for menus over 26 options, --cuda-graphs OFF. The author asked on 6 Oct 2026 that the TRAINED version replace the untrained Hopper 12B request in GitHub #164; this is that row. NOT the published `hopper` row (HopitAI, rank 25), a different system. One RTX 6000 Ada 48 GB (Lium, Toronto); the author ran theirs on an L40S. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. add-requests run 54, GitHub #164",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 6000 Ada",
   "intelligence": {
    "I_open": 52.49257010742919,
    "I_sealed": 50.822792838986025,
    "base": 51.6576814732076,
    "excess": -0.9036103739950812,
    "gap": 1.6697772684431627,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 68.64489681634264,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 60.36489462094999,
       "judge": 72.31377719182596,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 43.547273982056595,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 45.285539523888346,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.928801818275204,
       "hard": 41.02383720746134,
       "judge": 39.409476186258296,
       "standard": 53.30140801599398
      }
     },
     "sealed|choice": {
      "cc": 65.07976853625175,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 82.70637794693668,
       "hard": 58.2579185520362,
       "judge": 62.756052141527014,
       "standard": 73.39573839142753
      }
     },
     "sealed|noul": {
      "cc": 41.18694322295551,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 18.446601941747566,
       "judge": 39.47368421052631,
       "standard": 71.83098591549295
      }
     },
     "sealed|score": {
      "cc": 46.20166675775083,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 67.66271785418321,
       "hard": 37.07866936088858,
       "judge": 45.777923536915246,
       "standard": 54.3527508345125
      }
     }
    }
   },
   "jevbench_score": 68.78245633029094,
   "key": "hopper-12b-trained",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "HopitAI/hopper@4b605f25eb903c2cf4d121040ffb097240b3bc37",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9319727891156463,
    "decisive_rate": 0.7924528301886793,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.63452107883388,
     "cost": 56.15749203571245,
     "intelligence": 51.6576814732076,
     "speed": 84.57301814857726
    },
    "board_release": "v1.6.43",
    "capability": 68.14610127602074,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 31,
    "capability_rank": 52,
    "composite_ci95": {
     "A": [
      54.67384317203267,
      67.43166618883266
     ],
     "B": [
      51.03564669331923,
      64.73449720897486
     ],
     "C": [
      37.9679466472449,
      58.605012542436775
     ]
    },
    "jevbench_score": 65.78419365065008,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 26,
    "rank": 47,
    "ranks": {
     "A": 47,
     "B": 49,
     "C": 54,
     "capability": 89
    },
    "scores": {
     "A": 65.78419365065008,
     "B": 62.37284887464997,
     "C": 48.76282692619866
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4776301250443794,
     "p50_s_raw": 0.1638150625221897,
     "p95_s_adjusted": 0.7304768792411778,
     "p95_s_raw": 0.2902384396205889
    },
    "views": {
     "Balanced 33:33:33": 61.23777421409552,
     "Emphasis on Accuracy 60:20:20": 57.0087849790455,
     "Emphasis on Cost 20:20:60": 59.09921236284679,
     "Emphasis on Speed 20:60:20": 68.83489755597157,
     "Intelligence only": 51.6576814732076
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 87,
   "ranked": true,
   "ranks": {
    "A": 87,
    "B": 97,
    "C": 101,
    "capability": 115
   },
   "repo": "https://huggingface.co/HopitAI/hopper",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.78245633029094,
    "B": 64.50566729185212,
    "C": 50.98530250905721
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5167318890220486,
    "p50_s_raw": 0.18336594451102428,
    "p95_s_adjusted": 0.8370608537457883,
    "p95_s_raw": 0.34353042687289415
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.81171527967304,
      87.00046880053556
     ],
     "I": [
      46.53255595156734,
      55.86954644502632
     ],
     "capability": [
      64.05753668009696,
      70.56574900316099
     ],
     "composite": null,
     "seed": 59
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.63452107883388,
     "capability": 68.14610127602074,
     "intelligence": 51.6576814732076
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7413333333333333,
      "calibration": 77.2373238787563,
      "cc": 66.92022773130735,
      "cc_by_type": {
       "choice": 66.92022773130735
      },
      "cc_tiered": 66.8623326762972,
      "cc_tiered_open": 68.64489681634264,
      "cc_tiered_sealed": 65.07976853625175,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 66.92022773130735
     },
     "noul": {
      "accuracy": 0.7306666666666667,
      "calibration": 88.72462948027713,
      "cc": 46.13333333333334,
      "cc_by_type": {
       "noul": 46.13333333333334
      },
      "cc_tiered": 42.36710860250605,
      "cc_tiered_open": 43.547273982056595,
      "cc_tiered_sealed": 41.18694322295551,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 46.13333333333334
     },
     "score": {
      "accuracy": 0.5973333333333334,
      "calibration": 87.94160987746824,
      "cc": 46.77672583471599,
      "cc_by_type": {
       "score": 46.77672583471599
      },
      "cc_tiered": 45.74360314081959,
      "cc_tiered_open": 45.285539523888346,
      "cc_tiered_sealed": 46.20166675775083,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 46.77672583471599
     }
    },
    "raw_on_A": {
     "calibration": 82.16875295682286,
     "capability": 68.22397555466662,
     "intelligence": 54.279198152510396
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.74048708777325,
    "Emphasis on Accuracy 60:20:20": 58.78530872787772,
    "Emphasis on Cost 20:20:60": 65.4552343422139,
    "Emphasis on Speed 20:60:20": 71.17344321368013,
    "Intelligence only": 51.6576814732076
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Hyland",
   "axes": {
    "calibration": 75.8316875668352,
    "cost": 49.09977911142268,
    "intelligence": 52.865450899914414,
    "speed": 93.51379976299526
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14317514920634938,
      "ece_score": 71.36497015873012,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 70.41343722328038,
      "score_before_time_limit": 71.36497015873012,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.16800000000000004,
      "brier": 0.12771623160380524,
      "ece": 0.09925250399999994,
      "n": 375,
      "score": 79.72203520426667,
      "score_before_time_limit": 80.14949920000001,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9066666666666666,
      "interval90_width": 0.3264444444444444,
      "n": 375,
      "nrps": 0.09940238260432994,
      "score": 77.35959027295856,
      "score_before_time_limit": 77.98345793645015,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.1704642293333334
     }
    },
    "score": 75.8316875668352
   },
   "capability": 64.3485692333748,
   "composite_ci95": {
    "A": [
     52.374566728540906,
     62.42181266526091
    ],
    "B": [
     49.592754478501206,
     60.69955947070597
    ],
    "C": [
     36.37122689482008,
     55.479920762925325
    ]
   },
   "cost": {
    "basis": "ESTIMATE, v1.5-M2 base-model reference rule. The customer states a benchmark tariff of USD 0.056 per million input tokens, output free (one NVIDIA L40S, AWS on-demand); there is no public price page for it, so it is not a published list price. Under the method, the base-model reference price applies as an ESTIMATE for this vendor-hosted row, as for attest-31b-v1.0: base model google/gemma-4-26b-a4b-it through the frozen pricing_v15 module (sha256 1f4d1f7b1ce97ab27b938043444b9c2757e4a9dbecc799a803122588401f5bf1) reference() -> google/gemma-4-26b-a4b-it on OpenRouter, USD 0.09 per million input tokens, USD 0.30 per million output tokens. The customer-stated tariff is shown for information and is not used for the Cost axis. Applied to this row's own provider-reported tokens, first answer per item (2,012,340 input and 0 output tokens over the 1,500 items; 820,650 input and 0 output tokens over the 1,485 items of the common cost basis; the 15 very long items count for no row's cost, whether the row refused or answered them). Estimated, not a vendor bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them in full).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.1207404
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.04973636363636364
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "HS1 0.1.2 (Hyland)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "paid evaluation. Vendor-hosted Hyland API (customer-supplied HTTPS endpoint, model `hyland-hs1`), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) over the official per-item latency subset of the first-occurrence rows, without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `hyland` (no proxy) before the first request. 1,500 of 1,500 unique items answered with HTTP 200 and the served model was `hyland-hs1` on every row; option-letter log-probability readout, nothing generated (0 output tokens). Run with the customer's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 60.80592519588509,
    "I_sealed": 44.92497660394374,
    "base": 52.865450899914414,
    "excess": 6.041421821585637,
    "gap": 15.880948591941348,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 64.83372028845844,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 60.36489462094998,
       "judge": 68.35860250494397,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 61.1207729468599,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 19.999999999999996,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 56.46328235233694,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.266396969696956,
       "hard": 57.13894351851854,
       "judge": 57.104464406779655,
       "standard": 54.24862962962963
      }
     },
     "sealed|choice": {
      "cc": 63.179889701235524,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 79.03430749682339,
       "hard": 56.80582861107898,
       "judge": 62.09598432908915,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 36.31329247181023,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 72.41379310344827,
       "hard": 34.54545454545455,
       "judge": 19.999999999999996,
       "standard": 46.26865671641791
      }
     },
     "sealed|score": {
      "cc": 35.281747638785475,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 62.096657894736836,
       "hard": 26.543902535211284,
       "judge": 26.121538541666666,
       "standard": 53.09029636363637
      }
     }
    }
   },
   "jevbench_score": 61.068034493527705,
   "key": "hyland-hs1",
   "last_measured_on": "2026-10-21",
   "licence": "Proprietary hosted API (no weights release); declared base Gemma 4 26B-A4B-it is under the Gemma 4 licence",
   "listing": "ranked",
   "measured_in": "v1.6.36",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:hyland.com/hyland-hs1/v1/systemone#hyland-hs1",
   "noul_decisive": {
    "acc_among_decisive": 0.8838709677419355,
    "decisive": 310,
    "decisive_rate": 0.8310991957104558,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 24,
    "axes": {
     "calibration": 76.49930909839343,
     "cost": 49.09977911142268,
     "intelligence": 53.57948799459325,
     "speed": 93.74328417303443
    },
    "board_release": "v1.6.43",
    "capability": 65.03939854649335,
    "capability_api_board_rank": 21,
    "capability_open_board_rank": null,
    "capability_rank": 59,
    "composite_ci95": {
     "A": [
      54.8992123734169,
      62.833909581191186
     ],
     "B": [
      52.24566392674325,
      61.163125168542486
     ],
     "C": [
      38.12445303709507,
      57.62480456059248
     ]
    },
    "jevbench_score": 61.450719939462154,
    "measured_in": "v1.6.36",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 56,
    "ranks": {
     "A": 56,
     "B": 54,
     "C": 53,
     "capability": 101
    },
    "scores": {
     "A": 61.450719939462154,
     "B": 59.20850453421004,
     "C": 49.002878617843415
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.19411396997747943,
     "p50_s_raw": 0.19411396997747943,
     "p95_s_adjusted": 0.21757790509145708,
     "p95_s_raw": 0.21757790509145708
    },
    "views": {
     "Balanced 33:33:33": 58.21053424780427,
     "Emphasis on Accuracy 60:20:20": 55.404055858567006,
     "Emphasis on Cost 20:20:60": 53.31752802618845,
     "Emphasis on Speed 20:60:20": 67.87813214465525,
     "Intelligence only": 51.66752129933851
    }
   },
   "rank": 108,
   "ranked": true,
   "ranks": {
    "A": 108,
    "B": 109,
    "C": 108,
    "capability": 133
   },
   "repo": "https://www.hyland.com",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 61.068034493527705,
    "B": 58.74291323238438,
    "C": 47.40840267556635
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19862511445535347,
    "p50_s_raw": 0.19862511445535347,
    "p95_s_adjusted": 0.2241743211168796,
    "p95_s_raw": 0.2241743211168796
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API: no customer code ran locally; weights and serving code are not public. Customer statements (not independently verified): HS1 0.1.2 is a LoRA fine-tune of Gemma-4-26B-A4B-it served in FP8 on one NVIDIA L40S (AWS on-demand); trained on \"10,762 curated rows\". DUPLICATE REQUESTS, FIRST ANSWER SCORED: because of an operator error two dispatches of the same driver overlapped briefly, so 64 items (items 2 to 65 of the run order) were requested twice; the first answer per item is scored, the 64 repeat rows are not scored, and the repeats carry no new exposure beyond the admitted full-set run. FULL LENGTH: the endpoint accepted every item, including the 15 longest (reported input 76,572 to 81,749 tokens), within its 131,072-token context; nothing was refused or truncated. MODEL, customer statements (unverified): LoRA fine-tune of Gemma-4-26B-A4B-it served in FP8; training on \"10,762 curated rows\"; weights and code are not public (API entrant). PUBLIC ITEMS: the customer was asked whether public JevBench items were used in training or model selection and had not answered when this release was built, so the normal public-vs-sealed gap rule applies without any claim either way.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 67.35632222222223,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 10.402299999999997
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 67.35632222222223
      },
      "calibration_gen": {
       "accuracy": 0.6435643564356436,
       "cc": 46.96054865897844,
       "cc_by_type": {
        "choice": 65.00530222693531,
        "noul": 60.00000000000001,
        "score": 15.876343749999988
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 46.96054865897844
      },
      "composite_gen": {
       "accuracy": 0.6732673267326733,
       "cc": 52.270568278785355,
       "cc_by_type": {
        "choice": 77.23577235772358,
        "noul": 55.55555555555556,
        "score": 24.020376923076913
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 52.270568278785355
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 80.64695185185185,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 81.94085555555557
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 80.64695185185185
      },
      "jaggedness": {
       "accuracy": 0.5277777777777778,
       "cc": 26.260480774082126,
       "cc_by_type": {
        "choice": 45.13955726660251,
        "noul": 2.7027027027026973,
        "score": 30.939182352941174
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 26.260480774082126
      },
      "llm_router": {
       "accuracy": 0.7859424920127795,
       "cc": 69.77902567381767,
       "cc_by_type": {
        "choice": 66.71656247248393,
        "noul": 79.38144329896907,
        "score": 63.23907124999999
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 69.77902567381767
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5925925925925926,
       "cc": 29.34662629684176,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": -25,
        "score": 50.15328095238094
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 29.34662629684176
      },
      "multilingual": {
       "accuracy": 0.6727272727272727,
       "cc": 50.017188940458446,
       "cc_by_type": {
        "choice": 61.85197261167266,
        "noul": 36.111111111111114,
        "score": 52.08848309859155
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 50.017188940458446
      },
      "negation_pair_gen": {
       "accuracy": 0.7674418604651163,
       "cc": 39.70408444444444,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 19.999999999999996,
        "score": 5.7789199999999985
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 39.70408444444444
      },
      "option_order": {
       "accuracy": 0.75,
       "cc": 68.04260985352863,
       "cc_by_type": {
        "choice": 68.04260985352863
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 68.04260985352863
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 79.1600528105028,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100,
        "score": 67.51018846153846
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 79.1600528105028
      },
      "usecase": {
       "accuracy": 0.6,
       "cc": 43.19437939607844,
       "cc_by_type": {
        "choice": 62.00980392156864,
        "noul": 29.166666666666675,
        "score": 38.40666760000001
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 43.19437939607844
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 36.60442454212454,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": 16.95613076923076
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 36.60442454212454
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 45.20500871479774,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 23.01901999999998
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 45.20500871479774
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 77.65046457611902,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 100,
        "score": 80.7390043478261
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 77.65046457611902
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 38.651429019242705,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 99.56694999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 38.651429019242705
      },
      "en": {
       "accuracy": 0.6794092093831451,
       "cc": 50.40503834349743,
       "cc_by_type": {
        "choice": 64.63308966207391,
        "noul": 47.482014388489205,
        "score": 39.100010979929166
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 50.40503834349743
      },
      "es": {
       "accuracy": 0.7096774193548387,
       "cc": 53.85744628794422,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 100,
        "score": -9.063876470588216
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 53.85744628794422
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 55.90507650273224,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 99.95566666666666
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 55.90507650273224
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 67.4717750257998,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 65.94473684210527
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 67.4717750257998
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 57.72511666666666,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 25,
        "score": 48.17534999999998
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 57.72511666666666
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 60.872964135396934,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 10.678128571428546
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 60.872964135396934
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 35.41934512046597,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 39.99999999999999,
        "score": 23.446279999999998
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 35.41934512046597
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 38.989697089947086,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 61.41353571428571
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 38.989697089947086
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.76121111111111,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.9503
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.76121111111111
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 27.763858239538234,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 68.57296
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 27.763858239538234
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 45.71841414900911,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 19.999999999999996,
        "score": 40.00754883720929
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 45.71841414900911
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 72.995288488224,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 50.8067258064516
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 72.995288488224
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 22.163285106382986,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 23.085600000000017
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 22.163285106382986
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 55.01890000000001,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 15.056700000000035
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 55.01890000000001
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.50802365591399,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 91.04020000000001
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.50802365591399
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 33.86654595103578,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": 60.3567
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 33.86654595103578
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7253333333333334,
       "cc": 64.52285730687538,
       "cc_by_type": {
        "choice": 64.52285730687538
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 64.52285730687538
      },
      "noul": {
       "accuracy": 0.7306666666666667,
       "cc": 46.13333333333334,
       "cc_by_type": {
        "noul": 46.13333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 46.13333333333334
      },
      "score": {
       "accuracy": 0.5466666666666666,
       "cc": 41.569475090252716,
       "cc_by_type": {
        "score": 41.569475090252716
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 41.569475090252716
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -11.300584313725494,
       "cc_by_type": {
        "choice": 36,
        "noul": -50,
        "score": -19.901752941176486
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 81.0909294871795,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 43.27278846153847
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 81.0909294871795
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 18.76733739837398,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": -44.91750000000001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 18.76733739837398
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 0.1842885233918056,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -33.333333333333336,
        "score": 4.061637499999982
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0.1842885233918056
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 35.30437397342996,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 68.23196250000001
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 35.30437397342996
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -55.67849392033543,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 0,
        "score": -204.1424
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 39.19570028729439,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 32.713151282051264
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 39.19570028729439
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 5.0591438569913665,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": -100,
        "score": 56.98927826086957
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 5.0591438569913665
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 13.626879629629627,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 19.999999999999996,
        "score": 4.213972222222218
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 13.626879629629627
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 46.05044599095885,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 35.72450000000001
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 46.05044599095885
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 44.080392655234526,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 55.55555555555556,
        "score": -0.058563636363628824
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 44.080392655234526
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": 16.414429256272395,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 44.94221249999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 16.414429256272395
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 57.32517348048626,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 60.00000000000001,
        "score": 35.00814999999999
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 57.32517348048626
      },
      "model_routing": {
       "accuracy": 0.8387096774193549,
       "cc": 75.62815107097764,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 93.93939393939394,
        "score": 53.9840203125
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.62815107097764
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 16.569089191223515,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 61.22358235294117
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 16.569089191223515
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 51.46014982977406,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 54.473300000000016
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 51.46014982977406
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 16.829311600502272,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": 0,
        "score": 9.60123529411765
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 16.829311600502272
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 25.47528971014493,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 53.23746333333334
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 25.47528971014493
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 93.97886359493039,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 92.61274736842105
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 93.97886359493039
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      70.79674006801244,
      78.35115191592271
     ],
     "I": [
      47.34948055006876,
      56.61223115738562
     ],
     "capability": [
      59.653393276541806,
      66.81231879137462
     ],
     "composite": {
      "A": [
       52.374566728540906,
       62.42181266526091
      ],
      "B": [
       49.592754478501206,
       60.69955947070597
      ],
      "C": [
       36.37122689482008,
       55.479920762925325
      ]
     },
     "seed": 30
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 75.8316875668352,
     "capability": 64.3485692333748,
     "intelligence": 52.865450899914414
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7253333333333334,
      "calibration": 70.41343722328038,
      "cc": 64.52285730687538,
      "cc_by_type": {
       "choice": 64.52285730687538
      },
      "cc_tiered": 64.00680499484699,
      "cc_tiered_open": 64.83372028845844,
      "cc_tiered_sealed": 63.179889701235524,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 64.52285730687538
     },
     "noul": {
      "accuracy": 0.7306666666666667,
      "calibration": 79.72203520426667,
      "cc": 46.13333333333334,
      "cc_by_type": {
       "noul": 46.13333333333334
      },
      "cc_tiered": 48.717032709335065,
      "cc_tiered_open": 61.1207729468599,
      "cc_tiered_sealed": 36.31329247181023,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 46.13333333333334
     },
     "score": {
      "accuracy": 0.5466666666666666,
      "calibration": 77.35959027295856,
      "cc": 41.569475090252716,
      "cc_by_type": {
       "score": 41.569475090252716
      },
      "cc_tiered": 45.872514995561204,
      "cc_tiered_open": 56.46328235233694,
      "cc_tiered_sealed": 35.281747638785475,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 41.569475090252716
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 57.88642123418364,
    "Emphasis on Accuracy 60:20:20": 54.91036265742412,
    "Emphasis on Cost 20:20:60": 53.153968766440535,
    "Emphasis on Speed 20:60:20": 67.56365834205323,
    "Intelligence only": 50.97896438738072
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "mohit67890",
   "axes": {
    "calibration": 86.77653179414553,
    "cost": 67.34930355763724,
    "intelligence": 21.160522003324633,
    "speed": 81.2429913982843
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03246566765774192,
      "ece_score": 93.5068664684516,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 93.5068664684516,
      "score_before_time_limit": 93.5068664684516,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.6246648793565683,
      "brier": 0.2028168457960074,
      "ece": 0.11038035259514468,
      "n": 373,
      "score": 77.92392948097107,
      "score_before_time_limit": 77.92392948097107,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9408602150537635,
      "interval90_width": 0.6883960573476698,
      "n": 372,
      "nrps": 0.1322491683106928,
      "score": 88.89879943301392,
      "score_before_time_limit": 88.89879943301392,
      "time_limit_failed_share": 0,
      "top_ece": 0.044887421514514296
     }
    },
    "score": 86.77653179414553
   },
   "capability": 53.96852689873508,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 3,010,517 input + 0 output tokens over 1,485 decisions = 2027.28417508 tokens/decision. GPU-time cost USD 0.0122562200029394/1,000; old list-price cost USD 0.0405456835016835/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0122562200029394/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.040140226666666674
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.012256220002939449
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Imajev-2B (mohit67890, Qwen3.5-2B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own scripts/playground/server.py --backend torch, 4 option orders, 4,096-token request limit.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 24.18657872532247,
    "I_sealed": 18.134465281326797,
    "base": 21.160522003324633,
    "excess": -3.7874133263600385,
    "gap": 6.0521134439956725,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 49.471301249374704,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 37.71626297577854,
       "judge": 52.537903757415954,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -42.85714285714286,
       "hard": -13.043478260869568,
       "judge": -60,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 23.08843492659271,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.907855488528355,
       "hard": 14.056431441245543,
       "judge": 33.77791221517532,
       "standard": 8.208515683445306
      }
     },
     "sealed|choice": {
      "cc": 41.84862177374901,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 54.2566709021601,
       "hard": 42.23430091361164,
       "judge": 38.29578844270325,
       "standard": 40.20248892638684
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -27.586206896551722,
       "hard": -49.09090909090909,
       "judge": -63.07692307692307,
       "standard": -7.462686567164178
      }
     },
     "sealed|score": {
      "cc": 12.554774070231389,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 26.48562308030278,
       "hard": 9.945290312389155,
       "judge": 9.940140408699815,
       "standard": 14.730267573177514
      }
     }
    }
   },
   "jevbench_score": 8.336534540273085,
   "key": "imajev-2b",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0 (adapter; base Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "mohit67890/imajev-2b@66e4b8c808bae45ca98bed5edd2de6323bdb0576",
   "noul_decisive": {
    "acc_among_decisive": 0.8428571428571429,
    "decisive": 140,
    "decisive_rate": 0.3753351206434316,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.77653179414553,
     "cost": 51.76166122103225,
     "intelligence": 21.160522003324633,
     "speed": 81.55302418750827
    },
    "board_release": "v1.6.43",
    "capability": 53.96852689873508,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 87,
    "capability_rank": 113,
    "composite_ci95": {
     "A": [
      4.658485929408603,
      10.94035382726261
     ],
     "B": [
      3.706707978661138,
      9.206275249274212
     ],
     "C": [
      3.2350596732004195,
      7.5974679355990355
     ]
    },
    "jevbench_score": 7.92834166805135,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": 138,
    "rank": 177,
    "ranks": {
     "A": 177,
     "B": 178,
     "C": 179,
     "capability": 168
    },
    "scores": {
     "A": 7.92834166805135,
     "B": 6.507269614912114,
     "C": 5.50579282503566
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.7288981560035609,
     "p50_s_raw": 0.28944907800178044,
     "p95_s_adjusted": 0.9594685189804295,
     "p95_s_raw": 0.40473425949021474
    },
    "views": {
     "Balanced 33:33:33": 6.815416220156846,
     "Emphasis on Accuracy 60:20:20": 5.165912913746609,
     "Emphasis on Cost 20:20:60": 7.623022614150632,
     "Emphasis on Speed 20:60:20": 8.663982835133574,
     "Intelligence only": 3.789999234949207
    }
   },
   "rank": 221,
   "ranked": true,
   "ranks": {
    "A": 221,
    "B": 222,
    "C": 222,
    "capability": 203
   },
   "repo": "https://huggingface.co/mohit67890/imajev-2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 8.336534540273085,
    "B": 6.723430553263221,
    "C": 5.789260097411865
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.745621624001069,
    "p50_s_raw": 0.2978108120005345,
    "p95_s_adjusted": 1.0073545804945752,
    "p95_s_raw": 0.42867729024728757
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7272727272727273,
       "cc": 27.940174431020967,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 50,
        "score": -32.846143373603766
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 27.940174431020967
      },
      "calibration_gen": {
       "accuracy": 0.297029702970297,
       "cc": -17.5634975525963,
       "cc_by_type": {
        "choice": 14.103923647932142,
        "noul": -60,
        "score": -6.7944163057210405
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.43564356435643564,
       "cc": -4.661934044854566,
       "cc_by_type": {
        "choice": 38.211382113821145,
        "noul": -62.96296296296296,
        "score": 10.765778714578122
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.6458333333333334,
       "cc": 52.590783388621844,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 37.772350165865554
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 52.590783388621844
      },
      "jaggedness": {
       "accuracy": 0.3277777777777778,
       "cc": -7.850740882891453,
       "cc_by_type": {
        "choice": 17.70933589990377,
        "noul": -40.54054054054054,
        "score": -0.7210180080375883
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.5175718849840255,
       "cc": 18.658985210953194,
       "cc_by_type": {
        "choice": 51.65976930527428,
        "noul": -23.71134020618557,
        "score": 28.02852653377088
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 18.658985210953194
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -18.44398052388307,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75,
        "score": -6.105137447937858
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5309090909090909,
       "cc": 16.31443135528656,
       "cc_by_type": {
        "choice": 56.9611998695794,
        "noul": -36.111111111111114,
        "score": 28.09320530739138
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 16.31443135528656
      },
      "negation_pair_gen": {
       "accuracy": 0.36046511627906974,
       "cc": -14.889109686015951,
       "cc_by_type": {
        "choice": 29.999999999999996,
        "noul": -73.33333333333334,
        "score": -1.3339957247145096
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.5,
       "cc": 36.08521970705726,
       "cc_by_type": {
        "choice": 36.08521970705726
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 36.08521970705726
      },
      "perturb_pair": {
       "accuracy": 0.6875,
       "cc": 52.302256042037875,
       "cc_by_type": {
        "choice": 57.95795795795795,
        "noul": 39.99999999999999,
        "score": 58.948810168155674
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 52.302256042037875
      },
      "usecase": {
       "accuracy": 0.4727272727272727,
       "cc": 8.637486944515624,
       "cc_by_type": {
        "choice": 53.43137254901962,
        "noul": -45.833333333333336,
        "score": 18.31442161786059
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 8.637486944515624
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -28.225026395455945,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": -50,
        "score": -58.48460299589164
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": -10.526315789473685,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 24.065748870216918,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 26.267907132924172
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 24.065748870216918
      },
      "de": {
       "accuracy": 0.71875,
       "cc": 42.362310149525264,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 0,
        "score": 43.016133988398806
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 42.362310149525264
      },
      "el": {
       "accuracy": 0.4,
       "cc": 19.38235434668214,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 91.85283585196449
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 19.38235434668214
      },
      "en": {
       "accuracy": 0.44917463075586445,
       "cc": 5.074350761629012,
       "cc_by_type": {
        "choice": 40.24969185145407,
        "noul": -38.12949640287769,
        "score": 13.10285683631066
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 5.074350761629012
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 46.5562811978586,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 50,
        "score": 19.032628259154937
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 46.5562811978586
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -10.63743944027403,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": 33.661452170981185
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 3.2263733482564874,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": -100,
        "score": 52.03206122124004
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 3.2263733482564874
      },
      "hi": {
       "accuracy": 0.42857142857142855,
       "cc": -10.475686348831847,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100,
        "score": 18.852270562443007
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.4666666666666667,
       "cc": -19.711858509492984,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": -100,
        "score": -31.076339363241235
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 34.29447363096327,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 19.999999999999996,
        "score": 25.77460437184137
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 34.29447363096327
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -9.900639833853115,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 14.742524942885105
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.6428571428571429,
       "cc": 58.50507211449983,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 58.84854967683284
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 58.50507211449983
      },
      "nl": {
       "accuracy": 0.4,
       "cc": 3.7552735572627327,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -33.333333333333336,
        "score": 22.521231927199448
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 3.7552735572627327
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5128205128205128,
       "cc": 16.728251852982496,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -40,
        "score": 13.03706194912947
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 16.728251852982496
      },
      "pt": {
       "accuracy": 0.39285714285714285,
       "cc": 20.212949326324164,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": 19.999999999999996,
        "score": 14.887522109791584
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 20.212949326324164
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": -0.25603227399320855,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -100,
        "score": 50.29573296525442
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 28.348191067910236,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50,
        "score": 0.2619645080785271
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 28.348191067910236
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 3.1976636096176585,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": 74.10911986111104
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 3.1976636096176585
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 8.401209830979823,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": 9.384420453391451
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 8.401209830979823
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5666666666666667,
       "cc": 44.02877973172085,
       "cc_by_type": {
        "choice": 44.02877973172085
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 44.02877973172085
      },
      "noul": {
       "accuracy": 0.31466666666666665,
       "cc": -37.06666666666667,
       "cc_by_type": {
        "noul": -37.06666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.42133333333333334,
       "cc": 15.622459826305535,
       "cc_by_type": {
        "score": 15.622459826305535
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 15.622459826305535
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.09090909090909091,
       "cc": -33.193862534650656,
       "cc_by_type": {
        "choice": 4,
        "noul": -100,
        "score": -3.5815876039519523
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": -6.089719732637479,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -55.55555555555556,
        "score": -6.8129825243444575
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.9285714285714286,
       "cc": 62.03940267844437,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 36.11820803533312
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 62.03940267844437
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -37.42857709364494,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -30.414386251695056
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 10.87022836857408,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 9.422279308620796
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 10.87022836857408
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -8.016826814669251,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -33.333333333333336,
        "score": -2.6668326452656155
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 9.059322026345937,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 28.018302213491587
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 9.059322026345937
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -15.154690545329686,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100,
        "score": 27.706660071328017
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": -11.501806643589282,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -100,
        "score": 27.994580069232157
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": -10.82046302383224,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": -33.333333333333336,
        "score": -9.192295566857167
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.44,
       "cc": -1.191894960640904,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -77.77777777777779,
        "score": -14.170000127400728
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -25.461994188704736,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -11.869853533856144
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.37037037037037035,
       "cc": -0.9809923099182641,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -60,
        "score": 3.1222821873277407
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.5576036866359447,
       "cc": 21.130055825784556,
       "cc_by_type": {
        "choice": 63.76623376623377,
        "noul": -24.242424242424242,
        "score": 23.86635795354414
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 21.130055825784556
      },
      "moderation_trust_safety": {
       "accuracy": 0.4444444444444444,
       "cc": -13.619363650781176,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100,
        "score": 16.72348294400964
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 53.4817305604329,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 43.824950269970955
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 53.4817305604329
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": -1.6382323766226665,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -100,
        "score": 24.64195311643741
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 9.317909661738637,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 33.75083043449128
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 9.317909661738637
      },
      "search_retrieval": {
       "accuracy": 0.5833333333333334,
       "cc": 21.221925502780437,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": -50,
        "score": 45.694246259230994
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 21.221925502780437
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.44171405018487,
      86.64641382932699
     ],
     "I": [
      17.21228402122313,
      24.114657957043782
     ],
     "capability": [
      49.4688678734668,
      54.186102023350564
     ],
     "composite": null,
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.77653179414553,
     "capability": 53.96852689873508,
     "intelligence": 21.160522003324633
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5666666666666667,
      "calibration": 93.5068664684516,
      "cc": 44.02877973172085,
      "cc_by_type": {
       "choice": 44.02877973172085
      },
      "cc_tiered": 45.65996151156186,
      "cc_tiered_open": 49.471301249374704,
      "cc_tiered_sealed": 41.84862177374901,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 44.02877973172085
     },
     "noul": {
      "accuracy": 0.31466666666666665,
      "calibration": 77.92392948097107,
      "cc": -37.06666666666667,
      "cc_by_type": {
       "noul": -37.06666666666667
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.42133333333333334,
      "calibration": 88.89879943301392,
      "cc": 15.622459826305535,
      "cc_by_type": {
       "score": 15.622459826305535
      },
      "cc_tiered": 17.82160449841205,
      "cc_tiered_open": 23.08843492659271,
      "cc_tiered_sealed": 12.554774070231389,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 15.622459826305535
     }
    },
    "raw_on_A": {
     "calibration": 83.85821528875951,
     "capability": 52.090117601921726,
     "intelligence": 20.322019915083942
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 7.220649287598299,
    "Emphasis on Accuracy 60:20:20": 5.301216757755804,
    "Emphasis on Cost 20:20:60": 8.60178123278197,
    "Emphasis on Speed 20:60:20": 9.042885429298314,
    "Intelligence only": 3.789999234949207
   }
  },
  {
   "adapter_id": "A2 reviewed bundle",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 89.1627567462182,
    "cost": 77.80651098778299,
    "intelligence": 34.62714018309005,
    "speed": 90.38312320077796
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04836725385924713,
      "ece_score": 90.32654922815058,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 90.32654922815058,
      "score_before_time_limit": 90.32654922815058,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.477088948787062,
      "brier": 0.12273935128941174,
      "ece": 0.06563823281458846,
      "n": 371,
      "score": 86.87235343708231,
      "score_before_time_limit": 86.87235343708231,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.967479674796748,
      "interval90_width": 0.43699186991869926,
      "n": 369,
      "nrps": 0.08429976663999325,
      "score": 90.28936757342169,
      "score_before_time_limit": 90.28936757342169,
      "time_limit_failed_share": 0,
      "top_ece": 0.054956440945786535
     }
    },
    "score": 89.1627567462182
   },
   "capability": 61.89494846465412,
   "class": "unclassified",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 907,776 input + 0 output tokens over 1,624 decisions = 558.975369458 tokens/decision. GPU-time cost USD 0.00549266316679539/1,000; old list-price cost USD 0.0167692610837438/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00549266316679539/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005492663166795393
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Imajev-4B (RTX 5090)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX 5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 35.03758089292987,
    "I_sealed": 34.21669947325023,
    "base": 34.62714018309005,
    "excess": -1.752506222758607,
    "gap": 0.8208814196796368,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 66.47085555091229,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 58.47750865051904,
       "judge": 80.22412656558997,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 0.6756383712905452,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 4.347826086956519,
       "judge": -30.000000000000004,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 37.966248756586765,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.45615279420953,
       "hard": 43.92313144450569,
       "judge": 33.905546551444175,
       "standard": 24.898584669651413
      }
     },
     "sealed|choice": {
      "cc": 62.38689563101728,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 70.73387037173899,
       "hard": 53.676470588235276,
       "judge": 64.61824953445065,
       "standard": 72.28722749107035
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -30.097087378640776,
       "judge": 0,
       "standard": 7.042253521126751
      }
     },
     "sealed|score": {
      "cc": 40.26320278873341,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.176956160290466,
       "hard": 35.01688311771597,
       "judge": 46.365402771352585,
       "standard": 30.14566547106101
      }
     }
    }
   },
   "jevbench_score": 29.970850718248183,
   "key": "imajev-4b-rtx5090-a2",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9639175257731959,
    "decisive_rate": 0.522911051212938,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.1627567462182,
     "cost": 63.26458220826236,
     "intelligence": 34.62714018309005,
     "speed": 90.51878000442085
    },
    "board_release": "v1.6.43",
    "capability": 61.89494846465412,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 51,
    "capability_rank": 76,
    "composite_ci95": {
     "A": [
      21.33216708823923,
      38.39023119963257
     ],
     "B": [
      18.266955184224454,
      34.31649986004929
     ],
     "C": [
      14.814004922388358,
      26.659882777522615
     ]
    },
    "jevbench_score": 28.655751132526984,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 73,
    "rank": 105,
    "ranks": {
     "A": 105,
     "B": 106,
     "C": 115,
     "capability": 119
    },
    "scores": {
     "A": 28.655751132526984,
     "B": 25.0248976556334,
     "C": 19.899827175365957
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.29240404299926015,
     "p50_s_raw": 0.07120202149963006,
     "p95_s_adjusted": 0.3034859753679484,
     "p95_s_raw": 0.07674298768397421
    },
    "views": {
     "Balanced 33:33:33": 25.816712856140082,
     "Emphasis on Accuracy 60:20:20": 21.130055842968158,
     "Emphasis on Cost 20:20:60": 27.454793753723415,
     "Emphasis on Speed 20:60:20": 30.81252811725961,
     "Intelligence only": 16.607714361099795
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 167,
   "ranked": true,
   "ranks": {
    "A": 167,
    "B": 168,
    "C": 169,
    "capability": 149
   },
   "repo": "https://github.com/fstandhartinger/jevbench/issues/110",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 29.970850718248183,
    "B": 25.816312665748484,
    "C": 20.813090776561236
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.29422962296521293,
    "p50_s_raw": 0.07211481148260646,
    "p95_s_adjusted": 0.31117255417630074,
    "p95_s_raw": 0.08058627708815037
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.2060903767957,
      89.78727704768245
     ],
     "I": [
      30.707360525579823,
      39.19772341159948
     ],
     "capability": [
      58.11557152205388,
      63.38857993223046
     ],
     "composite": null,
     "seed": 42
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.1627567462182,
     "capability": 61.89494846465412,
     "intelligence": 34.62714018309005
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7146666666666667,
      "calibration": 90.32654922815058,
      "cc": 63.50994193041121,
      "cc_by_type": {
       "choice": 63.50994193041121
      },
      "cc_tiered": 64.4288755909648,
      "cc_tiered_open": 66.47085555091229,
      "cc_tiered_sealed": 62.38689563101728,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 63.50994193041121
     },
     "noul": {
      "accuracy": 0.49866666666666665,
      "calibration": 86.87235343708231,
      "cc": -0.2666666666666706,
      "cc_by_type": {
       "noul": -0.2666666666666706
      },
      "cc_tiered": 0.3378191856452726,
      "cc_tiered_open": 0.6756383712905452,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.56,
      "calibration": 90.28936757342169,
      "cc": 40.37192786146962,
      "cc_by_type": {
       "score": 40.37192786146962
      },
      "cc_tiered": 39.11472577266009,
      "cc_tiered_open": 37.966248756586765,
      "cc_tiered_sealed": 40.26320278873341,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 40.37192786146962
     }
    },
    "raw_on_A": {
     "calibration": 87.91523763851959,
     "capability": 61.35771088116867,
     "intelligence": 34.80018412381776
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 27.253211223878914,
    "Emphasis on Accuracy 60:20:20": 21.691528433239966,
    "Emphasis on Cost 20:20:60": 30.548651862525315,
    "Emphasis on Speed 20:60:20": 32.00701035142905,
    "Intelligence only": 16.607714361099795
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "mohit67890",
   "axes": {
    "calibration": 88.10935553643324,
    "cost": 50.455376685541566,
    "intelligence": 36.7327337780622,
    "speed": 79.23152050102456
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05597354899568825,
      "ece_score": 88.80529020086236,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 88.80529020086236,
      "score_before_time_limit": 88.80529020086236,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.4584450402144772,
      "brier": 0.13920542517421816,
      "ece": 0.08183623766154813,
      "n": 373,
      "score": 83.63275246769038,
      "score_before_time_limit": 83.63275246769038,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.5192652329749106,
      "n": 372,
      "nrps": 0.09346155028027302,
      "score": 91.89002394074701,
      "score_before_time_limit": 91.89002394074701,
      "time_limit_failed_share": 0,
      "top_ece": 0.034368985452393384
     }
    },
    "score": 88.10935553643324
   },
   "capability": 62.42104465724772,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 3,010,517 input + 0 output tokens over 1,485 decisions = 2027.28417508 tokens/decision. GPU-time cost USD 0.0448216091127408/1,000; old list-price cost USD 0.202728417508417/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0448216091127408/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.20070113333333328
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.044821609112740766
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Imajev-9B (mohit67890, Qwen3.5-9B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own scripts/playground/server.py --backend torch, 4 option orders, 4,096-token request limit.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.23991434770264,
    "I_sealed": 32.22555320842176,
    "base": 36.7327337780622,
    "excess": -0.8251656310748317,
    "gap": 9.01436113928088,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 67.89294454901062,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 62.25228059138094,
       "judge": 72.31377719182598,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 13.632160110420985,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 21.739130434782616,
       "judge": -9.999999999999998,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 42.19463838367629,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.06511169132162,
       "hard": 33.49205778853101,
       "judge": 43.74474924939493,
       "standard": 54.33939662156624
      }
     },
     "sealed|choice": {
      "cc": 61.96527273043063,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 54.72418179715507,
       "judge": 62.09598432908915,
       "standard": 70.57582788441259
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 20.68965517241379,
       "hard": -9.090909090909093,
       "judge": -38.46153846153846,
       "standard": 13.432835820895516
      }
     },
     "sealed|score": {
      "cc": 34.71138689483466,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 55.85407978179955,
       "hard": 30.051996837602136,
       "judge": 30.53981630846708,
       "standard": 39.716176445368625
      }
     }
    }
   },
   "jevbench_score": 30.400650379076136,
   "key": "imajev-9b",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0 (adapter; base Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "mohit67890/imajev-9b@7301600591665b941d324b1a297e097b56724f9b",
   "noul_decisive": {
    "acc_among_decisive": 0.9257425742574258,
    "decisive": 202,
    "decisive_rate": 0.5415549597855228,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.10935553643324,
     "cost": 30.792561090951693,
     "intelligence": 36.7327337780622,
     "speed": 79.48842001125843
    },
    "board_release": "v1.6.43",
    "capability": 62.42104465724772,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      7.172547616858205,
      12.721677918306504
     ],
     "B": [
      6.6259340523666275,
      12.223727353008137
     ],
     "C": [
      4.980935845040419,
      8.834498554379516
     ]
    },
    "jevbench_score": 9.790864242743115,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": 133,
    "rank": 172,
    "ranks": {
     "A": 172,
     "B": 170,
     "C": 173,
     "capability": 117
    },
    "scores": {
     "A": 9.790864242743115,
     "B": 9.23297819070672,
     "C": 6.7992112796827175
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.9162941940070596,
     "p50_s_raw": 0.3831470970035298,
     "p95_s_adjusted": 1.227787094612722,
     "p95_s_raw": 0.5388935473063611
    },
    "views": {
     "Balanced 33:33:33": 8.496190189886166,
     "Emphasis on Accuracy 60:20:20": 8.076430781084673,
     "Emphasis on Cost 20:20:60": 7.458272603841369,
     "Emphasis on Speed 20:60:20": 10.503863437577062,
     "Intelligence only": 7.519195208738335
    }
   },
   "rank": 165,
   "ranked": true,
   "ranks": {
    "A": 165,
    "B": 163,
    "C": 167,
    "capability": 148
   },
   "repo": "https://huggingface.co/mohit67890/imajev-9b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 30.400650379076136,
    "B": 27.470001787736805,
    "C": 21.111562763247314
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.9317987710295711,
    "p50_s_raw": 0.39089938551478554,
    "p95_s_adjusted": 1.2809312370838597,
    "p95_s_raw": 0.5654656185419299
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 52.430958850575394,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": -9.373790114940505
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 52.430958850575394
      },
      "calibration_gen": {
       "accuracy": 0.46534653465346537,
       "cc": 5.028160919179807,
       "cc_by_type": {
        "choice": 42.73594909862143,
        "noul": -36,
        "score": 8.34853365891799
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 5.028160919179807
      },
      "composite_gen": {
       "accuracy": 0.5841584158415841,
       "cc": 19.016879555076702,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -48.14814814814815,
        "score": 37.7191120166303
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 19.016879555076702
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 77.21353987562888,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 71.64061962688662
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 77.21353987562888
      },
      "jaggedness": {
       "accuracy": 0.4777777777777778,
       "cc": 8.891936687850714,
       "cc_by_type": {
        "choice": 46.58325312800771,
        "noul": -51.35135135135135,
        "score": 31.443908286895773
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 8.891936687850714
      },
      "llm_router": {
       "accuracy": 0.7987220447284346,
       "cc": 62.159239624897964,
       "cc_by_type": {
        "choice": 76.22611605177423,
        "noul": 54.639175257731964,
        "score": 55.612427565187716
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 62.159239624897964
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -26.109429454822063,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -100,
        "score": 8.269649779863697
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6327272727272727,
       "cc": 34.94749358869319,
       "cc_by_type": {
        "choice": 66.7427453537659,
        "noul": -8.333333333333337,
        "score": 46.43306874564702
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 34.94749358869319
      },
      "negation_pair_gen": {
       "accuracy": 0.46511627906976744,
       "cc": 0.2664008509962761,
       "cc_by_type": {
        "choice": 63.333333333333336,
        "noul": -66.66666666666667,
        "score": 4.132535886322164
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0.2664008509962761
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 77.05660729696058,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100,
        "score": 55.19384591490575
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 77.05660729696058
      },
      "usecase": {
       "accuracy": 0.5590909090909091,
       "cc": 31.572158461704316,
       "cc_by_type": {
        "choice": 60.78431372549021,
        "noul": 0,
        "score": 33.932161659622736
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 31.572158461704316
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -5.14602112844375,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0,
        "score": -39.24758719485506
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 40.62744831788267,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": 30.023665998194872
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 40.62744831788267
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 65.36973340580671,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 50,
        "score": 62.03840375724315
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 65.36973340580671
      },
      "el": {
       "accuracy": 0.4,
       "cc": 15.425167488428194,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100,
        "score": 96.55483207422313
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 15.425167488428194
      },
      "en": {
       "accuracy": 0.6194613379669852,
       "cc": 32.65410958205473,
       "cc_by_type": {
        "choice": 63.53473840934329,
        "noul": -0.7194244604316502,
        "score": 35.14701479725255
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 32.65410958205473
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 62.56797666185319,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 7.279786429279
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 62.56797666185319
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 18.91749265853117,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -100,
        "score": 89.53936322149515
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 18.91749265853117
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 47.06376202766997,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 83.54422725948051
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 47.06376202766997
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 9.1729321161986,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -75,
        "score": 52.798125957534346
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 9.1729321161986
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 63.89055758345746,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 5.701290832991235
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 63.89055758345746
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 29.244543319970006,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 19.999999999999996,
        "score": 10.62481343886158
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 29.244543319970006
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 2.3136682734769445,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 51.385449264875284
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 2.3136682734769445
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.97684526925495,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 97.59720247443153
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.97684526925495
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 7.280856601680536,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 47.81659577906757
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 7.280856601680536
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -14.516129032258064,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 40.43577295910621,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 0,
        "score": 36.542189804106606
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 40.43577295910621
      },
      "pt": {
       "accuracy": 0.6071428571428571,
       "cc": 48.520372352488316,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": 60.00000000000001,
        "score": 49.202837741024176
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 48.520372352488316
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 8.986079710155908,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 49.0858987049358
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 8.986079710155908
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 14.927301478010037,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": -11.739834696404671
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 14.927301478010037
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 86.27310580160078,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 58.81931740480236
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 86.27310580160078
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 19.69631508464502,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -7.577721412731608
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 19.69631508464502
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.72,
       "cc": 63.83398074972732,
       "cc_by_type": {
        "choice": 63.83398074972732
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 63.83398074972732
      },
      "noul": {
       "accuracy": 0.49866666666666665,
       "cc": -0.2666666666666706,
       "cc_by_type": {
        "noul": -0.2666666666666706
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.5546666666666666,
       "cc": 36.90290826999303,
       "cc_by_type": {
        "score": 36.90290826999303
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.90290826999303
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -11.75076322631999,
       "cc_by_type": {
        "choice": 52,
        "noul": -75,
        "score": -12.252289678959972
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 7.659686301813146,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": -13.597890714988448
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 7.659686301813146
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 68.11810035858215,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 100,
        "score": 69.39495148225052
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 68.11810035858215
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -25.53781816870939,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -6.438015909636929
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 28.50650905063489,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 47.838367731614795
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 28.50650905063489
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -0.967848961333846,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -33.333333333333336,
        "score": -6.6771317896619165
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 29.47687169859932,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 32.1280940873946
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 29.47687169859932
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -8.77246629290122,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100,
        "score": 25.947409483665673
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 11.06914599494948,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60,
        "score": 34.87410465151511
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 11.06914599494948
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 26.706378828218817,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 33.33333333333333,
        "score": 36.721563322629336
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 26.706378828218817
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 12.856796666407638,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -55.55555555555556,
        "score": 5.753852531522674
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 12.856796666407638
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": -11.06811721251573,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100,
        "score": -0.9462871214181545
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5185185185185185,
       "cc": 25.34251136401156,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 30.57647842984659
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 25.34251136401156
      },
      "model_routing": {
       "accuracy": 0.8202764976958525,
       "cc": 63.2648273073167,
       "cc_by_type": {
        "choice": 82.46753246753246,
        "noul": 63.63636363636365,
        "score": 43.690585818054
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 63.2648273073167
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 7.461255389755934,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 56.93271050707971
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 7.461255389755934
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 50.77332873916746,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 52.4128367281802
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.77332873916746
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 17.749833188153378,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50,
        "score": 32.80614981076555
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 17.749833188153378
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 30.59909361582899,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 0,
        "score": 49.768295340240584
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 30.59909361582899
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 77.04547999886454,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 81.13643999659361
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 77.04547999886454
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.50079697136124,
      89.18191902573092
     ],
     "I": [
      32.20568334756309,
      41.13745884594617
     ],
     "capability": [
      58.46454287020759,
      64.17133593511335
     ],
     "composite": null,
     "seed": 32
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.10935553643324,
     "capability": 62.42104465724772,
     "intelligence": 36.7327337780622
    },
    "per_type": {
     "choice": {
      "accuracy": 0.72,
      "calibration": 88.80529020086236,
      "cc": 63.83398074972732,
      "cc_by_type": {
       "choice": 63.83398074972732
      },
      "cc_tiered": 64.92910863972062,
      "cc_tiered_open": 67.89294454901062,
      "cc_tiered_sealed": 61.96527273043063,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 63.83398074972732
     },
     "noul": {
      "accuracy": 0.49866666666666665,
      "calibration": 83.63275246769038,
      "cc": -0.2666666666666706,
      "cc_by_type": {
       "noul": -0.2666666666666706
      },
      "cc_tiered": 6.8160800552104925,
      "cc_tiered_open": 13.632160110420985,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.5546666666666666,
      "calibration": 91.89002394074701,
      "cc": 36.90290826999303,
      "cc_by_type": {
       "score": 36.90290826999303
      },
      "cc_tiered": 38.453012639255476,
      "cc_tiered_open": 42.19463838367629,
      "cc_tiered_sealed": 34.71138689483466,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.90290826999303
     }
    },
    "raw_on_A": {
     "calibration": 83.37739283046191,
     "capability": 60.48742810868637,
     "intelligence": 37.59746338691083
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 27.137661129438094,
    "Emphasis on Accuracy 60:20:20": 23.64863643004554,
    "Emphasis on Cost 20:20:60": 27.175178466998393,
    "Emphasis on Speed 20:60:20": 31.782904358462282,
    "Intelligence only": 19.825298960900696
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "rayrain-srp (ZooWork)",
   "axes": {
    "calibration": 93.17,
    "cost": 64.41,
    "intelligence": 47.77,
    "speed": 87.14
   },
   "calibration": {
    "score": 93.17
   },
   "capability": 70.47,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     39.53,
     72.11
    ],
    "B": [
     35.42,
     68.6
    ],
    "C": [
     27.45,
     66.05
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.03/M input, 0/M output (spec instinct.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.015355329949238578
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Instinct (ZooWork, Qwen3.8-27B)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 50.26,
    "I_sealed": 40.14,
    "base": 47.77,
    "per_type_split": {
     "open|choice": {
      "cc": 76.6889,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.2162,
       "hard": 69.8018,
       "judge": 80.2241,
       "standard": 77.3964
      }
     },
     "open|noul": {
      "cc": 22.4893,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 21.7391,
       "judge": 10,
       "standard": 11.1111
      }
     },
     "open|score": {
      "cc": 51.5916,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.5514,
       "hard": 43.6306,
       "judge": 66.2661,
       "standard": 40.522
      }
     },
     "sealed|choice": {
      "cc": 69.9997,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 88.8786,
       "hard": 61.2194,
       "judge": 68.3417,
       "standard": 80.6076
      }
     },
     "sealed|noul": {
      "cc": 14.607,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 64.7059,
       "hard": 9.0909,
       "judge": -10,
       "standard": 37.5
      }
     },
     "sealed|score": {
      "cc": 35.8134,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 62.6995,
       "hard": 22.3344,
       "judge": 24.153,
       "standard": 66.8189
      }
     }
    }
   },
   "jevbench_score": 62.23,
   "key": "instinct",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary service over Apache-2.0 base weights (Qwen3.8-27B)",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 23,
    "axes": {
     "calibration": 93.17,
     "cost": 64.41,
     "intelligence": 47.77,
     "speed": 87.2
    },
    "board_release": "v1.6.43",
    "capability": 70.47,
    "capability_api_board_rank": 19,
    "capability_open_board_rank": null,
    "capability_rank": 45,
    "composite_ci95": {
     "A": [
      39.54,
      72.12
     ],
     "B": [
      35.42,
      68.61
     ],
     "C": [
      27.46,
      66.05
     ]
    },
    "jevbench_score": 62.24,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 55,
    "ranks": {
     "A": 55,
     "B": 58,
     "C": 61,
     "capability": 76
    },
    "scores": {
     "A": 62.24,
     "B": 57.34,
     "C": 43.22
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.292,
     "p50_s_raw": 0.292,
     "p95_s_adjusted": 0.652,
     "p95_s_raw": 0.652
    },
    "views": null
   },
   "rank": 107,
   "ranked": true,
   "ranks": {
    "A": 107,
    "B": 110,
    "C": 113,
    "capability": 95
   },
   "repo": "https://github.com/fstandhartinger/jevbench/issues/69",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 62.23,
    "B": 57.33,
    "C": 43.22
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.296,
    "p50_s_raw": 0.296,
    "p95_s_adjusted": 0.653,
    "p95_s_raw": 0.653
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "83e1676aa839fe8666ffd6623d701c421d5841083daf6792bd3e3ab4be9d6c5c"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 600,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "ZooWork / pierre-srp",
   "axes": {
    "calibration": 86.08,
    "cost": 75.03,
    "intelligence": 28.34,
    "speed": 88.56
   },
   "calibration": {
    "score": 86.08
   },
   "capability": 57.21,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     10.92,
     37.03
    ],
    "B": [
     8.89,
     32.67
    ],
    "C": [
     7.58,
     25.72
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.01/M input, 0/M output (spec instinct-dual-4b.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.006799306260575296
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Instinct Dual 4B",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 26.29,
    "I_sealed": 25.26,
    "base": 28.34,
    "per_type_split": {
     "open|choice": {
      "cc": 51.7127,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.8649,
       "hard": 45.2658,
       "judge": 52.5379,
       "standard": 54.7928
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.8571,
       "hard": -4.3478,
       "judge": -10,
       "standard": -11.1111
      }
     },
     "open|score": {
      "cc": 27.145,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.8556,
       "hard": 24.9911,
       "judge": 27.0981,
       "standard": 15.6677
      }
     },
     "sealed|choice": {
      "cc": 49.0204,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 55.5144,
       "hard": 43.7682,
       "judge": 45.7286,
       "standard": 61.2153
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 17.6471,
       "hard": 9.0909,
       "judge": -30,
       "standard": 0
      }
     },
     "sealed|score": {
      "cc": 26.7459,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 42.7071,
       "hard": 24.586,
       "judge": 5.175,
       "standard": 55.4417
      }
     }
    }
   },
   "jevbench_score": 17.97,
   "key": "instinct-dual-4b",
   "last_measured_on": "2026-10-06",
   "licence": "Not independently recorded",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 36,
    "axes": {
     "calibration": 86.08,
     "cost": 75.03,
     "intelligence": 28.34,
     "speed": 88.6
    },
    "board_release": "v1.6.43",
    "capability": 57.21,
    "capability_api_board_rank": 25,
    "capability_open_board_rank": null,
    "capability_rank": 93,
    "composite_ci95": {
     "A": [
      10.92,
      37.04
     ],
     "B": [
      8.89,
      32.67
     ],
     "C": [
      7.58,
      25.72
     ]
    },
    "jevbench_score": 17.97,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 142,
    "ranks": {
     "A": 142,
     "B": 150,
     "C": 146,
     "capability": 147
    },
    "scores": {
     "A": 17.97,
     "B": 15.04,
     "C": 12.48
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 600,
     "p50_s_adjusted": 0.231,
     "p50_s_raw": 0.231,
     "p95_s_adjusted": 0.598,
     "p95_s_raw": 0.598
    },
    "views": null
   },
   "rank": 193,
   "ranked": true,
   "ranks": {
    "A": 193,
    "B": 195,
    "C": 194,
    "capability": 179
   },
   "repo": "https://github.com/fstandhartinger/jevbench/issues/123",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 17.97,
    "B": 15.04,
    "C": 12.48
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.232,
    "p50_s_raw": 0.232,
    "p95_s_adjusted": 0.601,
    "p95_s_raw": 0.601
   },
   "status": {
    "answered_ok": 600,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "8d8a87d185d8fb317472c636b36cbb1f0ac8f03abef8c751bdfa290a9aacc746"
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Macrocosmos (paid evaluation, PUBLIC visibility)",
   "axes": {
    "calibration": 84.8346261602328,
    "cost": 77.51227090165808,
    "intelligence": 29.414053145614563,
    "speed": 87.14949384308518
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06930761904761903,
      "ece_score": 86.13847619047618,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 86.13847619047618,
      "score_before_time_limit": 86.13847619047618,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.4,
      "brier": 0.1728691851466666,
      "ece": 0.07378479999999997,
      "n": 375,
      "score": 85.24304000000001,
      "score_before_time_limit": 85.24304000000001,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.944,
      "interval90_width": 0.5526666666666668,
      "n": 375,
      "nrps": 0.12630688752888886,
      "score": 83.12236229022221,
      "score_before_time_limit": 83.12236229022221,
      "time_limit_failed_share": 0,
      "top_ece": 0.10562293333333339
     }
    },
    "score": 84.8346261602328
   },
   "capability": 57.12433965292368,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 849,038 input + 0 output tokens over 1,485 decisions = 571.742760943 tokens/decision. GPU-time cost USD 0.00561811946554373/1,000; old list-price cost USD 0.0171522828282828/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00561811946554373/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.021895960000000003
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005618119465543728
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "iota-decide-4b (Macrocosmos, Clef joint-head 4B, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA A100-SXM4-80GB (FIN-03, Finland, EU, pay-as-you-go non-spot, USD 2.141/h). Container with --network none (cut proven from inside before the sealed input), Hub offline, every credential unset, read-only weights, all 16 weight files sha256-verified; driven one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 21 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 36.85502187502523,
    "I_sealed": 21.973084416203896,
    "base": 29.414053145614563,
    "excess": 5.042410688465623,
    "gap": 14.881937458821334,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 59.050171703581924,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 58.47750865051904,
       "judge": 64.40342781806197,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 14.56866804692892,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": 21.739130434782616,
       "judge": 0,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 36.946225874564846,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.07090909090908,
       "hard": 34.7399074074074,
       "judge": 42.41847457627117,
       "standard": 28.08814814814813
      }
     },
     "sealed|choice": {
      "cc": 47.967773170565664,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 67.5984752223634,
       "hard": 42.75471261709262,
       "judge": 49.75514201762979,
       "standard": 45.89748998101666
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 17.24137931034482,
       "hard": -16.36363636363637,
       "judge": -32.30769230769231,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 17.95148007804603,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 30.493157894736854,
       "hard": 16.664225352112673,
       "judge": 10.76348958333333,
       "standard": 25.03713636363637
      }
     }
    }
   },
   "jevbench_score": 19.7303928088595,
   "key": "iota-decide-4b",
   "last_measured_on": "2026-10-21",
   "licence": "Apache-2.0 (HF metadata, LICENSE, NOTICE; README notes CC-BY-SA NLI training records), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.38",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "macrocosm-os/iota-decide-4b@bd88855a7e3379468537e0142d5f0ada079aca41",
   "noul_decisive": {
    "acc_among_decisive": 0.84,
    "decisive": 225,
    "decisive_rate": 0.6,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.8346261602328,
     "cost": 62.97034212213745,
     "intelligence": 29.414053145614563,
     "speed": 87.4670144658746
    },
    "board_release": "v1.6.43",
    "capability": 57.12433965292368,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 70,
    "capability_rank": 96,
    "composite_ci95": {
     "A": [
      11.615032461140338,
      25.03863609150094
     ],
     "B": [
      9.632718447097313,
      21.808128076335468
     ],
     "C": [
      8.065994764680791,
      17.387941730208983
     ]
    },
    "jevbench_score": 18.937474020302744,
    "measured_in": "v1.6.38",
    "method": "v1.6",
    "open_board_rank": 100,
    "rank": 135,
    "ranks": {
     "A": 135,
     "B": 145,
     "C": 142,
     "capability": 150
    },
    "scores": {
     "A": 18.937474020302744,
     "B": 16.15725640426057,
     "C": 13.151023625210243
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3842653010000504,
     "p50_s_raw": 0.11713265050002519,
     "p95_s_adjusted": 0.46630211340004735,
     "p95_s_raw": 0.1581510567000237
    },
    "views": {
     "Balanced 33:33:33": 16.933803631615454,
     "Emphasis on Accuracy 60:20:20": 13.382059401682262,
     "Emphasis on Cost 20:20:60": 18.591825785082126,
     "Emphasis on Speed 20:60:20": 20.556488297647732,
     "Intelligence only": 10.17945694092124
    }
   },
   "rank": 187,
   "ranked": true,
   "ranks": {
    "A": 187,
    "B": 189,
    "C": 188,
    "capability": 182
   },
   "repo": "https://huggingface.co/macrocosm-os/iota-decide-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 19.7303928088595,
    "B": 16.612952550374235,
    "C": 13.7016616728191
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3936773030001405,
    "p50_s_raw": 0.12183865150007023,
    "p95_s_adjusted": 0.48967760110003833,
    "p95_s_raw": 0.16983880055001918
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "First-party host-authored HTTP /v1/systemone bridge (sha256 7c4c9c68a7bdc057) around the repo's documented load_release_model()/systemone() and its vendored joint_schema_model.py (Apache-2.0, from Cloudflare/clef); pre-measurement source review PASS and independent bridge review PASS; no trust_remote_code, no pickle weights. FIRST MEASUREMENT (paid evaluation). CONTEXT: states longer than 16,384 tokens are truncated by the model to 16,384 tokens and answered (15 items on this draw; scored as answered, as for earlier provider-truncated rows). BRIDGE: served through a first-party host-authored bridge around the author's documented entry points, not an author server. REPO: published as macrocosm-os/iota-sys1-4b, renamed to macrocosm-os/iota-decide-4b; measured revision bd88855a (weights/code/config identical at HEAD 745ccae4). PROVENANCE, author statements (unverified): LoRA fine-tune of Qwen/Qwen3.5-4B-Base distilled on Cloudflare Clef teacher probabilities (541,071 records: generated/template decision states plus Open-Jev, typed-decisions, civil_comments, GoEmotions, prompt-injections and 8,932 MNLI/SNLI/BoolQ records under CC-BY-SA); the README states no benchmark data was used and overlapping records were dropped. HARDWARE: A100 80 GB; latency reflects that card. Measured accuracy on the 15 shortened states: 2 of 15 correct (13.3 %), versus 822 of 1485 (55.4 %) on the remaining items; a Score item is correct within half a label step.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7272727272727273,
       "cc": 63.25333333333331,
       "cc_by_type": {
        "choice": 50,
        "noul": 100,
        "score": 39.75999999999995
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 63.25333333333331
      },
      "calibration_gen": {
       "accuracy": 0.38613861386138615,
       "cc": 4.616781990102507,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": -12,
        "score": 5.383749999999987
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 4.616781990102507
      },
      "composite_gen": {
       "accuracy": 0.5544554455445545,
       "cc": 19.422861047413896,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": -25.92592592592593,
        "score": 10.210769230769245
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 19.422861047413896
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 60.32685185185185,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 60.98055555555557
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 60.32685185185185
      },
      "jaggedness": {
       "accuracy": 0.3333333333333333,
       "cc": -12.641671961833303,
       "cc_by_type": {
        "choice": 20.596727622714162,
        "noul": -51.35135135135135,
        "score": -7.170392156862726
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.7028753993610224,
       "cc": 53.352812366788186,
       "cc_by_type": {
        "choice": 61.96178568283877,
        "noul": 50.51546391752577,
        "score": 47.58118750000002
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 53.352812366788186
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -39.16193548387097,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100,
        "score": -14.260000000000005
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -19.993658975617734,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -50,
        "score": -11.011904761904745
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5745454545454546,
       "cc": 25.22416665263471,
       "cc_by_type": {
        "choice": 57.93935441799805,
        "noul": -16.666666666666664,
        "score": 34.399812206572754
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 25.22416665263471
      },
      "negation_pair_gen": {
       "accuracy": 0.4069767441860465,
       "cc": -10.474999999999989,
       "cc_by_type": {
        "choice": 40.00000000000001,
        "noul": -60,
        "score": -11.424999999999974
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.6875,
       "cc": 60.05326231691079,
       "cc_by_type": {
        "choice": 60.05326231691079
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 60.05326231691079
      },
      "perturb_pair": {
       "accuracy": 0.75,
       "cc": 60.49063101563102,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 60.00000000000001,
        "score": 51.50192307692309
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 60.49063101563102
      },
      "usecase": {
       "accuracy": 0.5,
       "cc": 26.85113385620917,
       "cc_by_type": {
        "choice": 47.30392156862747,
        "noul": 12.5,
        "score": 20.74948000000003
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 26.85113385620917
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": -8.991672771672768,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 0,
        "score": -31.736923076923063
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 30.521888888888885,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 24.898999999999994
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 30.521888888888885
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 68.38896113889957,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 75,
        "score": 46.09608695652175
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 68.38896113889957
      },
      "el": {
       "accuracy": 0.4,
       "cc": 16.15022346368715,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100,
        "score": 98.73
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 16.15022346368715
      },
      "en": {
       "accuracy": 0.5412684622067767,
       "cc": 24.200852323574026,
       "cc_by_type": {
        "choice": 48.597161372206806,
        "noul": 2.877697841726623,
        "score": 21.127697756788642
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 24.200852323574026
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 69.97578863192912,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 19.715294117647055
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 69.97578863192912
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -18.429034608378867,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": 10.286666666666678
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 35.50656346749226,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 48.87263157894737
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 35.50656346749226
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 6.236473463687151,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 0,
        "score": -31.01125
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 6.236473463687151
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 63.882159373492186,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 19.705714285714304
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 63.882159373492186
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 15.028362898243758,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -19.999999999999996,
        "score": 22.273333333333344
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 15.028362898243758
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -10.676481481481494,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 12.414999999999976
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 92.53944444444444,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 94.285
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 92.53944444444444
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -15.770683982683984,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 4.635999999999996
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -39.516129032258064,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5384615384615384,
       "cc": 16.406287599195604,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -40,
        "score": 19.688604651162777
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 16.406287599195604
      },
      "pt": {
       "accuracy": 0.5357142857142857,
       "cc": 31.94592582736138,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": 19.999999999999996,
        "score": 50.08645161290322
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 31.94592582736138
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 12.416418439716315,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 73.845
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 12.416418439716315
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 12.944619565217392,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50,
        "score": -45.948750000000004
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 12.944619565217392
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 80.72064516129034,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 74.42000000000002
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 80.72064516129034
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -25.120263653484006,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -91.18000000000004
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6173333333333333,
       "cc": 50.57310702462733,
       "cc_by_type": {
        "choice": 50.57310702462733
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 50.57310702462733
      },
      "noul": {
       "accuracy": 0.504,
       "cc": 0.8000000000000007,
       "cc_by_type": {
        "noul": 0.8000000000000007
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.8000000000000007
      },
      "score": {
       "accuracy": 0.45866666666666667,
       "cc": 23.044314079422456,
       "cc_by_type": {
        "score": 23.044314079422456
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 23.044314079422456
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -4.71392156862746,
       "cc_by_type": {
        "choice": 4,
        "noul": -25,
        "score": 6.858235294117621
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": -19.2541912194086,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -33.333333333333336,
        "score": -31.26153846153843
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 40.176504065040646,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": 69.30999999999999
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 40.176504065040646
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -8.258944931773884,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": 2.123749999999991
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 21.14580314009663,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 25.756250000000016
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 21.14580314009663
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -18.243249475891,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -66.66666666666667,
        "score": -25.170000000000048
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 8.067751203045319,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 25.043589743589735
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 8.067751203045319
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -8.178912286017274,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -50,
        "score": 9.08695652173911
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": 18.254074074074072,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": 19.999999999999996,
        "score": 38.92888888888889
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.254074074074072
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": 11.594826098241517,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 0,
        "score": 11.872272727272726
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 11.594826098241517
      },
      "lead_generation": {
       "accuracy": 0.48,
       "cc": 13.114300368021299,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -55.55555555555556,
        "score": 6.526363636363652
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 13.114300368021299
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -47.04889784946236,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100,
        "score": -44.372499999999995
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5185185185185185,
       "cc": 46.48546918319471,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 60.00000000000001,
        "score": 25.521666666666643
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 46.48546918319471
      },
      "model_routing": {
       "accuracy": 0.7188940092165899,
       "cc": 52.40613658910534,
       "cc_by_type": {
        "choice": 68.44155844155844,
        "noul": 57.57575757575757,
        "score": 31.20109375
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 52.40613658910534
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": -11.69606676451771,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 10.977058823529406
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 41.228152393876634,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 23.777307692307716
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 41.228152393876634
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 22.454253839466833,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": -3.0805882352940905
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 22.454253839466833
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 11.62452173913043,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 40.670666666666655
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 11.62452173913043
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 70.2200530686146,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50,
        "score": 71.33631578947369
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 70.2200530686146
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.65934328285569,
      85.84677132978175
     ],
     "I": [
      24.25545772976716,
      32.98874141715469
     ],
     "capability": [
      52.41656740849436,
      58.60148516850051
     ],
     "composite": null,
     "seed": 34
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.8346261602328,
     "capability": 57.12433965292368,
     "intelligence": 29.414053145614563
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6173333333333333,
      "calibration": 86.13847619047618,
      "cc": 50.57310702462733,
      "cc_by_type": {
       "choice": 50.57310702462733
      },
      "cc_tiered": 53.508972437073794,
      "cc_tiered_open": 59.050171703581924,
      "cc_tiered_sealed": 47.967773170565664,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 50.57310702462733
     },
     "noul": {
      "accuracy": 0.504,
      "calibration": 85.24304000000001,
      "cc": 0.8000000000000007,
      "cc_by_type": {
       "noul": 0.8000000000000007
      },
      "cc_tiered": 7.28433402346446,
      "cc_tiered_open": 14.56866804692892,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.8000000000000007
     },
     "score": {
      "accuracy": 0.45866666666666667,
      "calibration": 83.12236229022221,
      "cc": 23.044314079422456,
      "cc_by_type": {
       "score": 23.044314079422456
      },
      "cc_tiered": 27.44885297630544,
      "cc_tiered_open": 36.946225874564846,
      "cc_tiered_sealed": 17.95148007804603,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 23.044314079422456
     }
    },
    "raw_on_A": {
     "calibration": 81.68506181215012,
     "capability": 55.999809766504924,
     "intelligence": 30.314557720859717
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 17.786005753992725,
    "Emphasis on Accuracy 60:20:20": 13.693150254217766,
    "Emphasis on Cost 20:20:60": 20.556744865928845,
    "Emphasis on Speed 20:60:20": 21.278005265659672,
    "Intelligence only": 10.17945694092124
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Macrocosmos (paid evaluation, PUBLIC visibility)",
   "axes": {
    "calibration": 84.75645357480421,
    "cost": 78.86867492076232,
    "intelligence": 52.532639982463,
    "speed": 87.46776808261981
   },
   "base_model": "Qwen/Qwen3.5-4B",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06229238095238098,
      "ece_score": 87.5415238095238,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.5415238095238,
      "score_before_time_limit": 87.5415238095238,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.15475363506666642,
      "ece": 0.10332613333333349,
      "n": 375,
      "score": 79.3347733333333,
      "score_before_time_limit": 79.3347733333333,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9733333333333334,
      "interval90_width": 0.5955555555555555,
      "n": 375,
      "nrps": 0.10106726170222215,
      "score": 87.39306358155557,
      "score_before_time_limit": 87.39306358155557,
      "time_limit_failed_share": 0,
      "top_ece": 0.07553573333333331
     }
    },
    "score": 84.75645357480421
   },
   "capability": 68.64454677863361,
   "code_revision": "First-party host-authored HTTP /v1/systemone bridge (sha256 9d23d050e537d9d7) around the repo's documented letter_decide.load_release_model()/systemone() (letter_decide.py blob 9d42370e); fast-lane source review PASS (review sha e7b48c5f) and independent bridge delta reviews PASS (kit SHAS 9673c3ec); no trust_remote_code, no pickle weights.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 765,092 input + 0 output tokens over 1,485 decisions = 515.213468013 tokens/decision. GPU-time cost USD 0.00506264532109491/1,000; old list-price cost USD 0.015456404040404/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00506264532109491/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.020178500000000002
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005062645321094913
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of v0.3 (paid evaluation). FAMILY: new checkpoint of a family with an earlier d1012 row (iota-decide-4b, v1.6.38); new weights (only the stock tokenizer is shared). CONTEXT: prompts longer than 16,384 tokens are cut by the model (letter_decide.fit_state keeps the head and tail of the state) and answered (15 items on this draw; scored as answered). READOUT: one forward pass per question; softmax over the option-letter logits with a stored temperature. The author's documented NOUL_FLOOR moves any yes/no probability inside 0.2-0.8 to 0.801/0.199 on the same side (applied to every input, answer unchanged). BRIDGE: served through a first-party host-authored bridge around the author's documented entry points, not an author server. PROVENANCE, author statements (unverified): LoRA r16 fine-tune of Qwen/Qwen3.5-4B (instruct), 211k records / 488k questions incl. human-labelled public datasets and teacher-agreed generated states; the README states no benchmark items or benchmark-shaped data were used. HARDWARE: A100 80 GB; latency reflects that card.",
   "display": "iota-decide-4b-v0.3 (Macrocosmos, letter-readout 4B, self-hosted)",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-26",
    "release": "v1.7.10",
    "results_sha256": "e550b1b765ce11f2e4d3bed3de9c59abde005e1e0f06c4a309d8a41bed77cf18"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA A100-SXM4-80GB (FIN-03, Finland, EU, pay-as-you-go non-spot, USD 2.215/h), instance id e352f0be-a001-4a77-83ff-8734642cf20f. Container with --network none (cut proven from inside before the sealed input), Hub offline, every credential unset, read-only weights, all 11 weight files sha256-verified; driven one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 26 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 59.54536505341423,
    "I_sealed": 45.519914911511776,
    "base": 52.532639982463,
    "excess": 4.1859233715467425,
    "gap": 14.025450141902454,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 68.69330988672641,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 66.02705253224286,
       "judge": 76.26895187870798,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 67.1207729468599,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 39.99999999999999,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 42.82201232665639,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.04181818181821,
       "hard": 36.16694444444446,
       "judge": 46.79610169491524,
       "standard": 46.56111111111109
      }
     },
     "sealed|choice": {
      "cc": 53.95920977698984,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 79.03430749682339,
       "hard": 42.75471261709262,
       "judge": 57.68854064642509,
       "standard": 58.23665893271462
      }
     },
     "sealed|noul": {
      "cc": 51.72930620588882,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 58.62068965517242,
       "hard": 45.45454545454546,
       "judge": 41.53846153846155,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 30.87122875165668,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 40.36675438596491,
       "hard": 30.66602816901409,
       "judge": 22.20895833333335,
       "standard": 39.52727272727273
      }
     }
    }
   },
   "jevbench_score": 72.7982392443752,
   "key": "iota-decide-4b-v0.3",
   "last_measured_on": "2026-10-26",
   "licence": "Apache-2.0 (HF metadata and README), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "macrocosm-os/iota-decide-4b-v0.3@1da87f01c949e2f8a65e5958ef7cc43ec18da999",
   "new_in": "v1.7.10",
   "noul_decisive": {
    "acc_among_decisive": 0.7866666666666666,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "rank": 50,
   "ranked": true,
   "ranks": {
    "A": 50,
    "B": 74,
    "C": 91,
    "capability": 111
   },
   "repo": "https://huggingface.co/macrocosm-os/iota-decide-4b-v0.3",
   "scores": {
    "A": 72.7982392443752,
    "B": 67.58385333595861,
    "C": 55.80547735159599
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3762809670000934,
    "p50_s_raw": 0.11314048350004668,
    "p95_s_adjusted": 0.4761139872001081,
    "p95_s_raw": 0.16305699360005405
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 53.086666666666666,
       "cc_by_type": {
        "choice": 50,
        "noul": 100,
        "score": 9.260000000000002
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 53.086666666666666
      },
      "calibration_gen": {
       "accuracy": 0.5445544554455446,
       "cc": 36.50387603835277,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": 68,
        "score": 8.319687499999983
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 36.50387603835277
      },
      "composite_gen": {
       "accuracy": 0.6534653465346535,
       "cc": 37.33522548815231,
       "cc_by_type": {
        "choice": 64.22764227642277,
        "noul": 11.111111111111116,
        "score": 36.66692307692306
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 37.33522548815231
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 86.81481481481482,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 60.44444444444444
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.81481481481482
      },
      "jaggedness": {
       "accuracy": 0.45555555555555555,
       "cc": 19.77927663382118,
       "cc_by_type": {
        "choice": 20.596727622714162,
        "noul": 29.729729729729737,
        "score": 9.011372549019637
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 19.77927663382118
      },
      "llm_router": {
       "accuracy": 0.8210862619808307,
       "cc": 71.1096313998595,
       "cc_by_type": {
        "choice": 72.26380206040326,
        "noul": 83.50515463917525,
        "score": 57.55993749999999
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 71.1096313998595
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": 12.166397849462369,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": 0,
        "score": 39.72500000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 12.166397849462369
      },
      "ml_usecase": {
       "accuracy": 0.5185185185185185,
       "cc": 14.84132220585829,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": 25,
        "score": 6.121904761904762
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 14.84132220585829
      },
      "multilingual": {
       "accuracy": 0.6945454545454546,
       "cc": 52.38216549572161,
       "cc_by_type": {
        "choice": 62.83012716009129,
        "noul": 52.77777777777777,
        "score": 41.53859154929578
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 52.38216549572161
      },
      "negation_pair_gen": {
       "accuracy": 0.6627906976744186,
       "cc": 33.507777777777775,
       "cc_by_type": {
        "choice": 53.333333333333336,
        "noul": 39.99999999999999,
        "score": 7.189999999999996
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 33.507777777777775
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.796875,
       "cc": 76.80934896434896,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100,
        "score": 60.458076923076916
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 76.80934896434896
      },
      "usecase": {
       "accuracy": 0.5636363636363636,
       "cc": 45.09834875816994,
       "cc_by_type": {
        "choice": 55.88235294117649,
        "noul": 45.83333333333333,
        "score": 33.57936
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 45.09834875816994
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 13.307020757020746,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50,
        "score": -33.88846153846157
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 13.307020757020746
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 54.17177777777778,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 29.181999999999984
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 54.17177777777778
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 69.9753982300885,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 58.82000000000001
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 69.9753982300885
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 37.18577901924271,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 95.17
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 37.18577901924271
      },
      "en": {
       "accuracy": 0.6550825369244135,
       "cc": 48.46736236762581,
       "cc_by_type": {
        "choice": 55.84627964022892,
        "noul": 56.83453237410072,
        "score": 32.7212750885478
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 48.46736236762581
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 69.40530275405432,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 27.79176470588236
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 69.40530275405432
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 41.897632058287805,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 57.93333333333335
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 41.897632058287805
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 41.10586171310629,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 65.67052631578947
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 41.10586171310629
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 50.77723230912477,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 50,
        "score": 35.851250000000014
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 50.77723230912477
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8,
       "cc": 67.14584159150799,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 15.467142857142836
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 67.14584159150799
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 53.81627217368282,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 80,
        "score": 24.340000000000007
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 53.81627217368282
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 41.39809523809524,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 24.194285714285712
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 41.39809523809524
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 94.87888888888888,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 84.63666666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 94.87888888888888
      },
      "nl": {
       "accuracy": 0.8,
       "cc": 68.031316017316,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 100,
        "score": 56.04199999999996
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 68.031316017316
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 64.2006976744186,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 32.6020930232558
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 64.2006976744186
      },
      "pt": {
       "accuracy": 0.6071428571428571,
       "cc": 58.41001184886675,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": 100,
        "score": 49.47870967741934
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 58.41001184886675
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 13.973085106382987,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 38.51500000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 13.973085106382987
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 53.25557971014492,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100,
        "score": 3.244999999999987
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 53.25557971014492
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 67.39962365591397,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 66.71499999999999
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.39962365591397
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 13.908625235404896,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": -40.76
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 13.908625235404896
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6706666666666666,
       "cc": 57.461872596107845,
       "cc_by_type": {
        "choice": 57.461872596107845
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 57.461872596107845
      },
      "noul": {
       "accuracy": 0.7866666666666666,
       "cc": 57.33333333333333,
       "cc_by_type": {
        "noul": 57.33333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 57.33333333333333
      },
      "score": {
       "accuracy": 0.5413333333333333,
       "cc": 34.31759927797842,
       "cc_by_type": {
        "score": 34.31759927797842
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 34.31759927797842
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 17.282941176470576,
       "cc_by_type": {
        "choice": 52,
        "noul": 0,
        "score": -0.15117647058826567
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 17.282941176470576
      },
      "code_linting": {
       "accuracy": 0.625,
       "cc": 40.55974801366106,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 55.55555555555556,
        "score": 3.3907692307692305
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 40.55974801366106
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 66.60655826558265,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 32.33999999999999
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 66.60655826558265
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 46.137999512670554,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 100,
        "score": 31.98124999999998
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 46.137999512670554
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 26.932980072463767,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": 33.33333333333333,
        "score": 19.929375000000004
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 26.932980072463767
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -24.5450104821803,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -33.333333333333336,
        "score": -64.83000000000001
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 47.20957049486461,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 28.183333333333316
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 47.20957049486461
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -12.03143059132455,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100,
        "score": 26.623478260869547
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 34.51462962962963,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 46.043888888888894
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 34.51462962962963
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 32.93977559319101,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 66.66666666666667,
        "score": 9.240454545454547
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 32.93977559319101
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 65.1395023882233,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 29.268636363636368
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 65.1395023882233
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 4.969345878136193,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -33.333333333333336,
        "score": 12.757499999999977
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 4.969345878136193
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 50.23843214615767,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 60.00000000000001,
        "score": 36.78055555555553
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 50.23843214615767
      },
      "model_routing": {
       "accuracy": 0.8387096774193549,
       "cc": 71.58647558170996,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 90.90909090909092,
        "score": 44.88929687500001
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 71.58647558170996
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 41.857606789356815,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 0,
        "score": 60.12176470588234
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 41.857606789356815
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 73.63729769302192,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 100,
        "score": 54.33807692307694
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 73.63729769302192
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 38.06349946875302,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": 28.96882352941178
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 38.06349946875302
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 42.88699516908213,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 66.66666666666667,
        "score": 19.965333333333323
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 42.88699516908213
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 72.52593026159705,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50,
        "score": 78.25394736842105
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 72.52593026159705
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.37070526022498,
      86.64927811084297
     ],
     "I": [
      47.45661388255614,
      55.82462371218084
     ],
     "capability": [
      64.3914989449463,
      70.57002396337161
     ],
     "composite": null,
     "seed": 33
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.75645357480421,
     "capability": 68.64454677863361,
     "intelligence": 52.532639982463
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6706666666666666,
      "calibration": 87.5415238095238,
      "cc": 57.461872596107845,
      "cc_by_type": {
       "choice": 57.461872596107845
      },
      "cc_tiered": 61.326259831858124,
      "cc_tiered_open": 68.69330988672641,
      "cc_tiered_sealed": 53.95920977698984,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.461872596107845
     },
     "noul": {
      "accuracy": 0.7866666666666666,
      "calibration": 79.3347733333333,
      "cc": 57.33333333333333,
      "cc_by_type": {
       "noul": 57.33333333333333
      },
      "cc_tiered": 59.42503957637436,
      "cc_tiered_open": 67.1207729468599,
      "cc_tiered_sealed": 51.72930620588882,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 57.33333333333333
     },
     "score": {
      "accuracy": 0.5413333333333333,
      "calibration": 87.39306358155557,
      "cc": 34.31759927797842,
      "cc_by_type": {
       "score": 34.31759927797842
      },
      "cc_tiered": 36.84662053915653,
      "cc_tiered_open": 42.82201232665639,
      "cc_tiered_sealed": 30.87122875165668,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 34.31759927797842
     }
    },
    "raw_on_A": {
     "calibration": 82.00840984057959,
     "capability": 68.50560598872751,
     "intelligence": 55.002802136875424
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.5283414977471,
    "Emphasis on Accuracy 60:20:20": 61.56161570130991,
    "Emphasis on Cost 20:20:60": 72.98579350736715,
    "Emphasis on Speed 20:60:20": 75.74215416899952,
    "Intelligence only": 52.532639982462996
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "theunnecessarythings",
   "axes": {
    "calibration": 50.5645403785324,
    "cost": 61.10443159149903,
    "intelligence": 47.93439876939806,
    "speed": 85.09218748397188
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14830087112828266,
      "ece_score": 70.33982577434347,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 70.33982577434347,
      "score_before_time_limit": 70.33982577434347,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.05660377358490565,
      "brier": 0.09551749322805621,
      "ece": 0.09323102319373136,
      "n": 371,
      "score": 81.35379536125373,
      "score_before_time_limit": 81.35379536125373,
      "time_limit_failed_share": 0
     }
    },
    "score": 50.5645403785324
   },
   "capability": 49.24946957396523,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 567,713 input + 1,108 output tokens over 1,477 decisions = 385.11916046 tokens/decision. GPU-time cost USD 0.019793333716611/1,000; old list-price cost USD 0.16368548408937/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.019793333716611/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.16117564
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.019793333716611017
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JADE (Qwen3.8-27B LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 47.17172129671271,
    "I_sealed": 48.6970762420834,
    "base": 47.93439876939806,
    "excess": -4.09874258780893,
    "gap": -1.5253549453706867,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 80.09487403506567,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 73.57659641396666,
       "judge": 80.22412656558997,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 61.42028985507247,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -163.63636363636363,
       "hard": -161.1111111111111,
       "judge": -213.55932203389827,
       "standard": -355.55555555555554
      }
     },
     "sealed|choice": {
      "cc": 75.66415221506249,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 70.47511312217195,
       "judge": 76.72253258845438,
       "standard": 82.26382559428504
      }
     },
     "sealed|noul": {
      "cc": 70.42707651118772,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 80,
       "hard": 57.28155339805825,
       "judge": 76.3157894736842,
       "standard": 83.09859154929578
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -182.97872340425533,
       "hard": -183.82352941176472,
       "judge": -204.79041916167668,
       "standard": -228.42639593908638
      }
     }
    }
   },
   "jevbench_score": 53.46928254382284,
   "key": "jade",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "theunnecessarythings/JADE@8e9f9a3aa4293190fcc4a6ff17ff5f3251f4c270",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9142857142857143,
    "decisive_rate": 0.9433962264150944,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 50.5645403785324,
     "cost": 33.57969498589053,
     "intelligence": 47.93439876939806,
     "speed": 85.53222225487117
    },
    "board_release": "v1.6.43",
    "capability": 49.24946957396523,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      16.72243182039576,
      22.540480675102366
     ],
     "B": [
      16.49103923273388,
      22.64957340042747
     ],
     "C": [
      11.612799875274835,
      16.71953364433383
     ]
    },
    "jevbench_score": 20.194459825016597,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 96,
    "rank": 131,
    "ranks": {
     "A": 131,
     "B": 129,
     "C": 139,
     "capability": 177
    },
    "scores": {
     "A": 20.194459825016597,
     "B": 20.12888689696233,
     "C": 14.023930434039306
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4045139710127842,
     "p50_s_raw": 0.12725698550639208,
     "p95_s_adjusted": 0.6915829036966897,
     "p95_s_raw": 0.27079145184834486
    },
    "views": {
     "Balanced 33:33:33": 19.951224387926274,
     "Emphasis on Accuracy 60:20:20": 19.918976230985706,
     "Emphasis on Cost 20:20:60": 17.004313740085063,
     "Emphasis on Speed 20:60:20": 24.181035896264405,
     "Intelligence only": 19.870798988937533
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 118,
   "ranked": true,
   "ranks": {
    "A": 118,
    "B": 118,
    "C": 125,
    "capability": 214
   },
   "repo": "https://huggingface.co/theunnecessarythings/JADE",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 53.46928254382284,
    "B": 51.277917699171454,
    "C": 37.13144621098809
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.42184902198496277,
    "p50_s_raw": 0.13592451099248137,
    "p95_s_adjusted": 0.7338785682804883,
    "p95_s_raw": 0.2919392841402441
   },
   "status": {
    "answered_ok": 1108,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      46.63888535444405,
      53.10214733728139
     ],
     "I": [
      44.307422428017674,
      51.72167160495215
     ],
     "capability": [
      45.932872400615175,
      51.86244666467468
     ],
     "composite": null,
     "seed": 47
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 50.5645403785324,
     "capability": 49.24946957396523,
     "intelligence": 47.93439876939806
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8186666666666667,
      "calibration": 70.33982577434347,
      "cc": 76.8100565539062,
      "cc_by_type": {
       "choice": 76.8100565539062
      },
      "cc_tiered": 77.87951312506408,
      "cc_tiered_open": 80.09487403506567,
      "cc_tiered_sealed": 75.66415221506249,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.8100565539062
     },
     "noul": {
      "accuracy": 0.8533333333333334,
      "calibration": 81.35379536125373,
      "cc": 70.66666666666667,
      "cc_by_type": {
       "noul": 70.66666666666667
      },
      "cc_tiered": 65.9236831831301,
      "cc_tiered_open": 61.42028985507247,
      "cc_tiered_sealed": 70.42707651118772,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 70.66666666666667
     },
     "score": {
      "accuracy": 0,
      "calibration": null,
      "cc": -197.36842105263145,
      "cc_by_type": {
       "score": -197.36842105263145
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 51.05198021321377,
     "capability": 50.51094734813176,
     "intelligence": 49.96991448304975
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 1,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 56.294226786101866,
    "Emphasis on Accuracy 60:20:20": 50.664445489974945,
    "Emphasis on Cost 20:20:60": 56.240467953594795,
    "Emphasis on Speed 20:60:20": 63.3997435635532,
    "Intelligence only": 44.05567348754112
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Icarus AI / cmxu",
   "axes": {
    "calibration": 88.12743013642095,
    "cost": 78.95653992094968,
    "intelligence": 37.57770128683752,
    "speed": 92.71898033409317
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08741578110266195,
      "ece_score": 82.5168437794676,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 82.5168437794676,
      "score_before_time_limit": 82.5168437794676,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.36657681940700804,
      "brier": 0.12068971845453581,
      "ece": 0.04093580710766913,
      "n": 371,
      "score": 91.81283857846617,
      "score_before_time_limit": 91.81283857846617,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9376693766937669,
      "interval90_width": 0.4548328816621501,
      "n": 369,
      "nrps": 0.10039054297386547,
      "score": 90.05260805132909,
      "score_before_time_limit": 90.05260805132909,
      "time_limit_failed_share": 0,
      "top_ece": 0.049278647999776395
     }
    },
    "score": 88.12743013642095
   },
   "capability": 62.85256571162924,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 831,083 input + 0 output tokens over 1,624 decisions = 511.750615764 tokens/decision. GPU-time cost USD 0.00502861827438687/1,000; old list-price cost USD 0.0153525184729064/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00502861827438687/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005028618274386871
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "janus 4B (Icarus AI / cmxu, Qwen3.5-4B + LoRA r64 and pointer decision head)",
   "endpoint_condition": "Current frozen 1500-item pool measured 2026-10-04 by coordinator plan P83 on one evaluator-owned Lium RTX PRO6000. Offline contributor image, serial loopback-only requests, no batching or retry; standard selfhosted x2+0.15 s latency adjustment. Current stage receipt verifies the same model pin used by the prior add-request run. Cost is the carried documented estimate recorded in current registry.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO6000",
   "intelligence": {
    "I_open": 40.660351519488046,
    "I_sealed": 34.49505105418699,
    "base": 37.57770128683752,
    "excess": 3.5919128228628145,
    "gap": 6.165300465301058,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 61.67923497711049,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 56.59012268008808,
       "judge": 60.44825313117996,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 23.289164941338854,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 47.82608695652173,
       "judge": -19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 37.01265464001479,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.99228809649041,
       "hard": 42.7176115493677,
       "judge": 46.3643071982838,
       "standard": 6.5854452556676435
      }
     },
     "sealed|choice": {
      "cc": 56.292971028212264,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 66.74303451333975,
       "hard": 49.60407239819003,
       "judge": 56.238361266294234,
       "standard": 64.52765118857003
      }
     },
     "sealed|noul": {
      "cc": 12.558893678884758,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48,
       "hard": -14.563106796116509,
       "judge": 23.684210526315795,
       "standard": 32.3943661971831
      }
     },
     "sealed|score": {
      "cc": 34.633288455463955,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 59.65725589324873,
       "hard": 29.365017382365,
       "judge": 35.41609562746013,
       "standard": 31.483636124775206
      }
     }
    }
   },
   "jevbench_score": 36.791451915443076,
   "key": "janus-4b",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "cmxu/janus-4b@fc86a8fbaa99de8dbac5aee46aaa56d1a853b876",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9446808510638298,
    "decisive_rate": 0.633423180592992,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.12743013642095,
     "cost": 64.41461114142905,
     "intelligence": 37.57770128683752,
     "speed": 93.17471817821269
    },
    "board_release": "v1.6.43",
    "capability": 62.85256571162924,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 46,
    "capability_rank": 71,
    "composite_ci95": {
     "A": [
      21.844531574541936,
      45.558507177419784
     ],
     "B": [
      18.72406374533524,
      41.13032827742159
     ],
     "C": [
      15.169813593431902,
      31.63785220654152
     ]
    },
    "jevbench_score": 35.18351981268845,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 63,
    "rank": 93,
    "ranks": {
     "A": 93,
     "B": 95,
     "C": 94,
     "capability": 113
    },
    "scores": {
     "A": 35.18351981268845,
     "B": 31.093845400105046,
     "C": 24.432999869922536
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2097848016768694,
     "p50_s_raw": 0.029892400838434696,
     "p95_s_adjusted": 0.22948492439463736,
     "p95_s_raw": 0.039742462197318675
    },
    "views": {
     "Balanced 33:33:33": 32.05124045782333,
     "Emphasis on Accuracy 60:20:20": 26.620100423360302,
     "Emphasis on Cost 20:20:60": 33.65415776578004,
     "Emphasis on Speed 20:60:20": 37.99318109571395,
     "Intelligence only": 21.225142796235538
    }
   },
   "provenance": {
    "carried_metadata_from": "run 47 author metadata + current P83 staged pin/measurement receipt",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 156,
   "ranked": true,
   "ranks": {
    "A": 156,
    "B": 156,
    "C": 157,
    "capability": 144
   },
   "repo": "https://huggingface.co/cmxu/janus-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 36.791451915443076,
    "B": 32.08525097581522,
    "C": 25.54961938572436
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2128553358837962,
    "p50_s_raw": 0.03142766794189811,
    "p95_s_adjusted": 0.2511987216770648,
    "p95_s_raw": 0.05059936083853242
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.8809685848649,
      88.36140314243593
     ],
     "I": [
      31.000587963045195,
      41.97868333470401
     ],
     "capability": [
      57.23309571773392,
      64.21484535464957
     ],
     "composite": null,
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.12743013642095,
     "capability": 62.85256571162924,
     "intelligence": 37.57770128683752
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6733333333333333,
      "calibration": 82.5168437794676,
      "cc": 58.22399893902218,
      "cc_by_type": {
       "choice": 58.22399893902218
      },
      "cc_tiered": 58.98610300266138,
      "cc_tiered_open": 61.67923497711049,
      "cc_tiered_sealed": 56.292971028212264,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 58.22399893902218
     },
     "noul": {
      "accuracy": 0.592,
      "calibration": 91.81283857846617,
      "cc": 18.399999999999995,
      "cc_by_type": {
       "noul": 18.399999999999995
      },
      "cc_tiered": 17.924029310111806,
      "cc_tiered_open": 23.289164941338854,
      "cc_tiered_sealed": 12.558893678884758,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.399999999999995
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 90.05260805132909,
      "cc": 36.47974838634844,
      "cc_by_type": {
       "score": 36.47974838634844
      },
      "cc_tiered": 35.82297154773937,
      "cc_tiered_open": 37.01265464001479,
      "cc_tiered_sealed": 34.633288455463955,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.47974838634844
     }
    },
    "raw_on_A": {
     "calibration": 84.32063965516178,
     "capability": null,
     "intelligence": null
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 33.84803550279342,
    "Emphasis on Accuracy 60:20:20": 27.343426474968275,
    "Emphasis on Cost 20:20:60": 37.459579616305106,
    "Emphasis on Speed 20:60:20": 39.42574995183591,
    "Intelligence only": 21.225142796235538
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "szybkie-ai",
   "axes": {
    "calibration": 89.98387597336676,
    "cost": 72.29693310055532,
    "intelligence": 33.22856158102424,
    "speed": 92.86051128678476
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.051101311475409826,
      "ece_score": 89.77973770491803,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 89.77973770491803,
      "score_before_time_limit": 89.77973770491803,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.4584450402144772,
      "brier": 0.1341961465415548,
      "ece": 0.047425201072386014,
      "n": 373,
      "score": 90.51495978552279,
      "score_before_time_limit": 90.51495978552279,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.5580197132616489,
      "n": 372,
      "nrps": 0.10785977850358425,
      "score": 89.6569304296595,
      "score_before_time_limit": 89.6569304296595,
      "time_limit_failed_share": 0,
      "top_ece": 0.0495008064516129
     }
    },
    "score": 89.98387597336676
   },
   "capability": 61.6062187771955,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 781,043 input + 1,485 output tokens over 1,485 decisions = 526.954882155 tokens/decision. GPU-time cost USD 0.00838369679687001/1,000; old list-price cost USD 0.0798932323232323/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00838369679687001/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0790943
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.008383696796870014
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JEB-35B-A3B (szybkie.ai, Qwen3.6-35B-A3B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's JEB server over vLLM (raw label probabilities, no calibration file, one presentation per question).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 38.44350210899075,
    "I_sealed": 28.013621053057726,
    "base": 33.22856158102424,
    "excess": 0.5903542855773125,
    "gap": 10.429881055933024,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 68.35657646052876,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 67.9144385026738,
       "judge": 52.537903757415954,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 12.366459627329192,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 39.13043478260869,
       "judge": -30.000000000000004,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 34.607470239114306,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 48.647878787878795,
       "hard": 27.61083333333334,
       "judge": 50.22610169491526,
       "standard": 18.152592592592566
      }
     },
     "sealed|choice": {
      "cc": 56.57345439924718,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 44.836359431016525,
       "judge": 58.570029382957905,
       "standard": 67.72832735709765
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 20.68965517241379,
       "hard": -10.909090909090914,
       "judge": -32.30769230769231,
       "standard": 19.402985074626855
      }
     },
     "sealed|score": {
      "cc": 27.467408759926002,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 49.020087719298246,
       "hard": 21.118253521126817,
       "judge": 25.38817708333333,
       "standard": 32.50822727272726
      }
     }
    }
   },
   "jevbench_score": 26.844904879645952,
   "key": "jeb-35b-a3b",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "szybkie-ai/jeb-35b-a3b@dfac7afaed21ffff638ae4588402db560376aeac",
   "noul_decisive": {
    "acc_among_decisive": 0.9356435643564357,
    "decisive": 202,
    "decisive_rate": 0.5415549597855228,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.98387597336676,
     "cost": 42.92470023353102,
     "intelligence": 33.22856158102424,
     "speed": 92.99325391268877
    },
    "board_release": "v1.6.43",
    "capability": 61.6062187771955,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      11.412397610637285,
      23.247151641339965
     ],
     "B": [
      9.926306265903968,
      21.251604442413498
     ],
     "C": [
      7.925276118498115,
      16.143855306486085
     ]
    },
    "jevbench_score": 17.300802214285724,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 109,
    "rank": 145,
    "ranks": {
     "A": 145,
     "B": 147,
     "C": 148,
     "capability": 127
    },
    "scores": {
     "A": 17.300802214285724,
     "B": 15.448393624283149,
     "C": 12.014445982142869
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.22059132121503353,
     "p50_s_raw": 0.035295660607516766,
     "p95_s_adjusted": 0.22755489745177326,
     "p95_s_raw": 0.03877744872588664
    },
    "views": {
     "Balanced 33:33:33": 15.22363233434469,
     "Emphasis on Accuracy 60:20:20": 13.089956907787629,
     "Emphasis on Cost 20:20:60": 14.69709047732901,
     "Emphasis on Speed 20:60:20": 19.00168433057001,
     "Intelligence only": 10.816061538150686
    }
   },
   "rank": 174,
   "ranked": true,
   "ranks": {
    "A": 174,
    "B": 175,
    "C": 176,
    "capability": 157
   },
   "repo": "https://huggingface.co/szybkie-ai/jeb-35b-a3b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 26.844904879645952,
    "B": 23.02613336908528,
    "C": 18.64229505530969
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2214831450022757,
    "p50_s_raw": 0.03574157250113785,
    "p95_s_adjusted": 0.2336728145368397,
    "p95_s_raw": 0.041836407268419855
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 59.72666666666665,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 4.179999999999939
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 59.72666666666665
      },
      "calibration_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 11.705334371686106,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": -4.0000000000000036,
        "score": 5.924062499999994
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 11.705334371686106
      },
      "composite_gen": {
       "accuracy": 0.5643564356435643,
       "cc": 19.52070356472796,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -33.333333333333336,
        "score": 24.415769230769236
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 19.52070356472796
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 84.24076719576719,
       "cc_by_type": {
        "choice": 90.47619047619048,
        "noul": 100,
        "score": 62.24611111111111
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.24076719576719
      },
      "jaggedness": {
       "accuracy": 0.4222222222222222,
       "cc": 0.657932106935687,
       "cc_by_type": {
        "choice": 30.702598652550538,
        "noul": -51.35135135135135,
        "score": 22.622549019607874
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.657932106935687
      },
      "llm_router": {
       "accuracy": 0.7412140575079872,
       "cc": 56.16464470490973,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 48.453608247422686,
        "score": 50.94637499999999
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 56.16464470490973
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": -7.445064637538863,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -75,
        "score": 14.520476190476206
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6218181818181818,
       "cc": 30.28596949640944,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": -13.888888888888884,
        "score": 38.98220657276996
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 30.28596949640944
      },
      "negation_pair_gen": {
       "accuracy": 0.5697674418604651,
       "cc": 10.81088888888889,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -46.666666666666664,
        "score": 5.765999999999993
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 10.81088888888889
      },
      "option_order": {
       "accuracy": 0.6666666666666666,
       "cc": 57.3901464713715,
       "cc_by_type": {
        "choice": 57.3901464713715
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 57.3901464713715
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 73.82135096635096,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 80,
        "score": 65.48807692307692
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 73.82135096635096
      },
      "usecase": {
       "accuracy": 0.5454545454545454,
       "cc": 21.5768768627451,
       "cc_by_type": {
        "choice": 62.00980392156864,
        "noul": -20.833333333333336,
        "score": 23.554160000000003
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 21.5768768627451
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 7.329047619047614,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0,
        "score": -20.87000000000001
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 7.329047619047614
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.5384615384615384,
       "cc": 5.657004096262161,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": -8.22099999999999
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 5.657004096262161
      },
      "de": {
       "accuracy": 0.75,
       "cc": 64.25974605617546,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 50,
        "score": 66.67304347826088
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 64.25974605617546
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 23.614667908131594,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 87.79
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 23.614667908131594
      },
      "en": {
       "accuracy": 0.5977410947002606,
       "cc": 29.215352744871566,
       "cc_by_type": {
        "choice": 57.38397139405179,
        "noul": 2.158273381294973,
        "score": 28.103813459267933
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 29.215352744871566
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 48.755200396635,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 50,
        "score": 6.053529411764713
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 48.755200396635
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 18.362003642987244,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 86.77999999999999
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 18.362003642987244
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 75.0780185758514,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 46.41052631578948
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 75.0780185758514
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 29.694657821229047,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -25,
        "score": 30.843749999999982
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 29.694657821229047
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 63.80406413539694,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 19.47142857142856
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 63.80406413539694
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 17.03858512046597,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 0,
        "score": 8.30399999999999
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 17.03858512046597
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -5.48076719576719,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 28.002142857142875
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.07833333333332,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 94.90166666666666
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.07833333333332
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 14.425538239538243,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 28.558000000000007
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 14.425538239538243
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 40.080546900373,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 0,
        "score": 35.47651162790697
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 40.080546900373
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 42.31917558499819,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 19.999999999999996,
        "score": 38.77838709677419
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 42.31917558499819
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 13.50039007092198,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -60,
        "score": 51.565
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 13.50039007092198
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 24.365416666666665,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": -26.903750000000002
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 24.365416666666665
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 77.10397849462366,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 63.570000000000014
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 77.10397849462366
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 18.364312617702463,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": 13.85000000000004
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 18.364312617702463
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6866666666666666,
       "cc": 59.528502267552,
       "cc_by_type": {
        "choice": 59.528502267552
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 59.528502267552
      },
      "noul": {
       "accuracy": 0.504,
       "cc": 0.8000000000000007,
       "cc_by_type": {
        "noul": 0.8000000000000007
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.8000000000000007
      },
      "score": {
       "accuracy": 0.552,
       "cc": 29.92095667870036,
       "cc_by_type": {
        "score": 29.92095667870036
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 29.92095667870036
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -23.629607843137265,
       "cc_by_type": {
        "choice": 52,
        "noul": -75,
        "score": -47.888823529411795
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": -0.203754047884478,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -33.333333333333336,
        "score": -11.37730769230767
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 31.193170731707323,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": 42.36000000000001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 31.193170731707323
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -30.005031676413257,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -8.143750000000004
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 10.044051932367147,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 6.9437499999999925
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 10.044051932367147
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -24.42324947589097,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -66.66666666666667,
        "score": -43.70999999999996
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 9.24208503914385,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -71.42857142857143,
        "score": 38.37051282051279
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 9.24208503914385
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": -9.992762207746297,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100,
        "score": 22.28652173913044
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 14.67074074074074,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 66.51222222222222
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 14.67074074074074
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 33.534732550342824,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 18.662727272727274
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 33.534732550342824
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 34.76682562054655,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 11.111111111111116,
        "score": 4.817272727272737
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 34.76682562054655
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -49.31033602150538,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -18.898750000000007
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5185185185185185,
       "cc": 7.846555591099727,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -60,
        "score": 18.088611111111085
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 7.846555591099727
      },
      "model_routing": {
       "accuracy": 0.7557603686635944,
       "cc": 56.35260518127705,
       "cc_by_type": {
        "choice": 76.62337662337664,
        "noul": 54.54545454545454,
        "score": 37.888984374999964
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 56.35260518127705
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 14.596378382447028,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 66.82176470588234
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 14.596378382447028
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 53.74200354736566,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 44.60576923076924
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 53.74200354736566
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 16.266214623780545,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50,
        "score": 28.355294117647045
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 16.266214623780545
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": -7.030033816425129,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -66.66666666666667,
        "score": 18.040333333333315
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 56.419999999999995,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 69.26
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 56.419999999999995
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.93418214562868,
      89.85283877896852
     ],
     "I": [
      28.037739089154297,
      37.67898728636531
     ],
     "capability": [
      56.857189752779114,
      62.91121122015222
     ],
     "composite": null,
     "seed": 33
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.98387597336676,
     "capability": 61.6062187771955,
     "intelligence": 33.22856158102424
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6866666666666666,
      "calibration": 89.77973770491803,
      "cc": 59.528502267552,
      "cc_by_type": {
       "choice": 59.528502267552
      },
      "cc_tiered": 62.46501542988797,
      "cc_tiered_open": 68.35657646052876,
      "cc_tiered_sealed": 56.57345439924718,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.528502267552
     },
     "noul": {
      "accuracy": 0.504,
      "calibration": 90.51495978552279,
      "cc": 0.8000000000000007,
      "cc_by_type": {
       "noul": 0.8000000000000007
      },
      "cc_tiered": 6.183229813664596,
      "cc_tiered_open": 12.366459627329192,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.8000000000000007
     },
     "score": {
      "accuracy": 0.552,
      "calibration": 89.6569304296595,
      "cc": 29.92095667870036,
      "cc_by_type": {
       "score": 29.92095667870036
      },
      "cc_tiered": 31.037439499520154,
      "cc_tiered_open": 34.607470239114306,
      "cc_tiered_sealed": 27.467408759926002,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 29.92095667870036
     }
    },
    "raw_on_A": {
     "calibration": 86.05549626571135,
     "capability": 59.931312510984895,
     "intelligence": 33.807128756258436
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 24.22448205273833,
    "Emphasis on Accuracy 60:20:20": 19.221699668528473,
    "Emphasis on Cost 20:20:60": 26.81280082731414,
    "Emphasis on Speed 20:60:20": 28.967452891874363,
    "Intelligence only": 14.675557769835786
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Frontier Infra",
   "axes": {
    "calibration": 83.00593897276424,
    "cost": 57.46143151439629,
    "intelligence": 57.4596907179922,
    "speed": 85.07834149803253
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13434975244299677,
      "ece_score": 73.13004951140066,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 73.03254277871879,
      "score_before_time_limit": 73.13004951140066,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.26415094339622647,
      "brier": 0.06648894231173322,
      "ece": 0.08077593261455521,
      "n": 371,
      "score": 83.84481347708896,
      "score_before_time_limit": 83.84481347708896,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.981029810298103,
      "interval90_width": 0.36359530261969286,
      "n": 369,
      "nrps": 0.05869267834921634,
      "score": 92.14046066248498,
      "score_before_time_limit": 92.14046066248498,
      "time_limit_failed_share": 0,
      "top_ece": 0.049249054200542
     }
    },
    "score": 83.00593897276424
   },
   "capability": 70.23281484537821,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 750,854 input + 1,477 output tokens over 1,477 decisions = 509.364251862 tokens/decision. GPU-time cost USD 0.0261789535694914/1,000; old list-price cost USD 0.21651298578199/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0261789535694914/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.21319311999999999
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.026178953569491424
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jebadiah 27B (Frontier Infra, Qwen3.8-27B LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 57.25677458890849,
    "I_sealed": 57.66260684707591,
    "base": 57.4596907179922,
    "excess": -2.9792199006056586,
    "gap": -0.40583225816741475,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 73.88416658400591,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 71.68921044353571,
       "judge": 68.35860250494396,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 37.36853002070393,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 60.51762716201564,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 69.48122121212121,
       "hard": 54.84767685185186,
       "judge": 70.97751186440678,
       "standard": 51.685903703703694
      }
     },
     "sealed|choice": {
      "cc": 76.61944220069019,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 67.42081447963801,
       "judge": 83.24022346368716,
       "standard": 83.37233649464221
      }
     },
     "sealed|noul": {
      "cc": 39.64965634378576,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 63.99999999999999,
       "hard": 18.446601941747566,
       "judge": 42.10526315789473,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 56.71872199675177,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.98117588652481,
       "hard": 44.34649191176467,
       "judge": 60.52794730538922,
       "standard": 69.61811725888323
      }
     }
    }
   },
   "jevbench_score": 68.25043421482714,
   "key": "jebadiah-27b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "frontier-infra/jebadiah-27b@4d3ed81640bbc015b47976bedae67212a5fb39cd",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9816849816849816,
    "decisive_rate": 0.7358490566037735,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.03844121699153,
     "cost": 29.935481526523958,
     "intelligence": 57.58551644935426,
     "speed": 85.83700487110255
    },
    "board_release": "v1.6.43",
    "capability": 70.3119788331729,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      18.744465890345797,
      19.52946206149045
     ],
     "B": [
      18.712637254320782,
      20.028302946408985
     ],
     "C": [
      13.997484201611632,
      19.52946206149045
     ]
    },
    "jevbench_score": 19.255307672764186,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 99,
    "rank": 134,
    "ranks": {
     "A": 134,
     "B": 131,
     "C": 124,
     "capability": 77
    },
    "scores": {
     "A": 19.255307672764186,
     "B": 19.517487118738075,
     "C": 17.736768346068793
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4285312689666171,
     "p50_s_raw": 0.13926563448330853,
     "p95_s_adjusted": 0.6085790472658118,
     "p95_s_raw": 0.2292895236329059
    },
    "views": {
     "Balanced 33:33:33": 17.22762964959432,
     "Emphasis on Accuracy 60:20:20": 18.448138764634205,
     "Emphasis on Cost 20:20:60": 13.86870216120643,
     "Emphasis on Speed 20:60:20": 20.908227606199645,
     "Intelligence only": 20.641713893950236
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 93,
   "ranked": true,
   "ranks": {
    "A": 93,
    "B": 88,
    "C": 80,
    "capability": 98
   },
   "repo": "https://huggingface.co/frontier-infra/jebadiah-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.25043421482714,
    "B": 65.77978805804823,
    "C": 62.59353598019311
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.463657653023256,
    "p50_s_raw": 0.156828826511628,
    "p95_s_adjusted": 0.6698358964873478,
    "p95_s_raw": 0.25991794824367387
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.4964804722844,
      84.29963743954265
     ],
     "I": [
      51.78610895497503,
      62.258853035805195
     ],
     "capability": [
      66.56436743169048,
      72.02289175876106
     ],
     "composite": null,
     "seed": 49
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.00593897276424,
     "capability": 70.23281484537821,
     "intelligence": 57.4596907179922
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8133333333333334,
      "calibration": 73.03254277871879,
      "cc": 76.12799939372697,
      "cc_by_type": {
       "choice": 76.12799939372697
      },
      "cc_tiered": 75.25180439234805,
      "cc_tiered_open": 73.88416658400591,
      "cc_tiered_sealed": 76.61944220069019,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.12799939372697
     },
     "noul": {
      "accuracy": 0.7146666666666667,
      "calibration": 83.84481347708896,
      "cc": 42.93333333333334,
      "cc_by_type": {
       "noul": 42.93333333333334
      },
      "cc_tiered": 38.50909318224484,
      "cc_tiered_open": 37.36853002070393,
      "cc_tiered_sealed": 39.64965634378576,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 42.93333333333334
     },
     "score": {
      "accuracy": 0.6853333333333333,
      "calibration": 92.14046066248498,
      "cc": 57.405502456140376,
      "cc_by_type": {
       "score": 57.405502456140376
      },
      "cc_tiered": 58.618174579383705,
      "cc_tiered_open": 60.51762716201564,
      "cc_tiered_sealed": 56.71872199675177,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 57.405502456140376
     }
    },
    "raw_on_A": {
     "calibration": 80.08029667613602,
     "capability": 71.54861127321053,
     "intelligence": 63.01692587028502
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01866666666666672,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.43249841909004,
    "Emphasis on Accuracy 60:20:20": 61.44969830833132,
    "Emphasis on Cost 20:20:60": 61.450494675569665,
    "Emphasis on Speed 20:60:20": 71.35914865425865,
    "Intelligence only": 57.4596907179922
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "frontier-infra",
   "axes": {
    "calibration": 83.31146464049283,
    "cost": 68.53174752675467,
    "intelligence": 30.5871791271758,
    "speed": 91.458678539542
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11935933114754105,
      "ece_score": 76.12813377049179,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 76.12813377049179,
      "score_before_time_limit": 76.12813377049179,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.48257372654155495,
      "brier": 0.14617931767116624,
      "ece": 0.06136039410187674,
      "n": 373,
      "score": 87.72792117962464,
      "score_before_time_limit": 87.72792117962464,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9865591397849462,
      "interval90_width": 0.6666666666666664,
      "n": 372,
      "nrps": 0.10575929584157615,
      "score": 86.07833897136206,
      "score_before_time_limit": 86.07833897136206,
      "time_limit_failed_share": 0,
      "top_ece": 0.08633696236559144
     }
    },
    "score": 83.31146464049283
   },
   "capability": 56.949321883834315,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 750,303 input + 1,485 output tokens over 1,485 decisions = 506.254545455 tokens/decision. GPU-time cost USD 0.0111928774598015/1,000; old list-price cost USD 0.0506754545454545/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0111928774598015/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05016869999999999
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011192877459801475
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jebadiah 9B v2 (Frontier Infra, Qwen3.5-9B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's standalone jebadiah_server: bf16, label-token logits, strict prompt contract, fitted per-type temperatures, fp32 candidate logits.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 34.34712859027669,
    "I_sealed": 26.827229664074913,
    "base": 30.5871791271758,
    "excess": -2.319627844153935,
    "gap": 7.519898926201776,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 63.94109602078727,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 58.47750865051904,
       "judge": 60.44825313117996,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": -30.434782608695656,
       "judge": -19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 39.1002897500428,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.8623575757576,
       "hard": 38.462709259259256,
       "judge": 35.66567627118642,
       "standard": 41.64633703703704
      }
     },
     "sealed|choice": {
      "cc": 53.67345673395262,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 71.41041931385007,
       "hard": 47.4384179484214,
       "judge": 52.39960822722822,
       "standard": 59.185825775152935
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 31.034482758620683,
       "hard": -12.727272727272732,
       "judge": -47.692307692307686,
       "standard": 7.462686567164178
      }
     },
     "sealed|score": {
      "cc": 26.808232258272128,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 43.64942105263159,
       "hard": 18.956308450704242,
       "judge": 24.474275000000013,
       "standard": 37.59242136363634
      }
     }
    }
   },
   "jevbench_score": 21.316898270250412,
   "key": "jebadiah-9b-v2",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "frontier-infra/jebadiah-9b-v2@bc63609af6434fdf18ed8b89e19df96db5a24944",
   "noul_decisive": {
    "acc_among_decisive": 0.9326424870466321,
    "decisive": 193,
    "decisive_rate": 0.517426273458445,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.31146464049283,
     "cost": 48.85607041331758,
     "intelligence": 30.5871791271758,
     "speed": 91.57148361254983
    },
    "board_release": "v1.6.43",
    "capability": 56.949321883834315,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 72,
    "capability_rank": 98,
    "composite_ci95": {
     "A": [
      14.321319372467606,
      25.03713540072477
     ],
     "B": [
      12.286822142404677,
      22.39679324818056
     ],
     "C": [
      9.945360675324725,
      17.38689958383665
     ]
    },
    "jevbench_score": 18.78430461690739,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 102,
    "rank": 137,
    "ranks": {
     "A": 137,
     "B": 144,
     "C": 143,
     "capability": 152
    },
    "scores": {
     "A": 18.78430461690739,
     "B": 16.423355223462213,
     "C": 13.044655983963466
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25783845791593196,
     "p50_s_raw": 0.05391922895796597,
     "p95_s_adjusted": 0.27008716338314115,
     "p95_s_raw": 0.06004358169157058
    },
    "views": {
     "Balanced 33:33:33": 16.727077379556174,
     "Emphasis on Accuracy 60:20:20": 13.798758909175735,
     "Emphasis on Cost 20:20:60": 17.01136530433605,
     "Emphasis on Speed 20:60:20": 20.79200680874768,
     "Intelligence only": 10.92887170611703
    }
   },
   "rank": 183,
   "ranked": true,
   "ranks": {
    "A": 183,
    "B": 185,
    "C": 185,
    "capability": 184
   },
   "repo": "https://huggingface.co/frontier-infra/jebadiah-9b-v2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.316898270250412,
    "B": 18.18139561422117,
    "C": 14.803401576562786
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2601211030036211,
    "p50_s_raw": 0.05506055150181055,
    "p95_s_adjusted": 0.2747619305271655,
    "p95_s_raw": 0.06238096526358276
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 61.07781111111111,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100,
        "score": 24.90009999999998
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 61.07781111111111
      },
      "calibration_gen": {
       "accuracy": 0.40594059405940597,
       "cc": 10.446769936373283,
       "cc_by_type": {
        "choice": 17.285259809119836,
        "noul": 12.00000000000001,
        "score": 2.0550500000000027
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 10.446769936373283
      },
      "composite_gen": {
       "accuracy": 0.5346534653465347,
       "cc": 18.094449115187732,
       "cc_by_type": {
        "choice": 70.73170731707317,
        "noul": -40.74074074074075,
        "score": 24.29238076923077
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 18.094449115187732
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 56.24330185185185,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 48.72990555555556
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 56.24330185185185
      },
      "jaggedness": {
       "accuracy": 0.36666666666666664,
       "cc": -1.3864631875152067,
       "cc_by_type": {
        "choice": 24.927815206929747,
        "noul": -45.945945945945944,
        "score": 16.85874117647058
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.7699680511182109,
       "cc": 62.21615746389665,
       "cc_by_type": {
        "choice": 70.67887646385489,
        "noul": 56.70103092783505,
        "score": 59.268564999999995
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 62.21615746389665
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -20.892842873506787,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -100,
        "score": 23.91940952380953
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6,
       "cc": 25.841182948220062,
       "cc_by_type": {
        "choice": 63.808281708509945,
        "noul": -25,
        "score": 38.71526713615023
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 25.841182948220062
      },
      "negation_pair_gen": {
       "accuracy": 0.4418604651162791,
       "cc": -6.473503333333329,
       "cc_by_type": {
        "choice": 56.66666666666668,
        "noul": -66.66666666666667,
        "score": -9.420509999999993
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 68.60171563101564,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 80,
        "score": 55.835176923076936
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 68.60171563101564
      },
      "usecase": {
       "accuracy": 0.4863636363636364,
       "cc": 18.583117720261452,
       "cc_by_type": {
        "choice": 49.75490196078433,
        "noul": -25,
        "score": 30.99445120000003
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 18.583117720261452
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 17.72341184371184,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50,
        "score": -1.5916692307692326
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 17.72341184371184
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 24.637396492575522,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 27.982849999999992
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 24.637396492575522
      },
      "de": {
       "accuracy": 0.75,
       "cc": 67.01662662562524,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75,
        "score": 57.908286956521735
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 67.01662662562524
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 24.128667908131593,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 89.33200000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 24.128667908131593
      },
      "en": {
       "accuracy": 0.5664639443961772,
       "cc": 26.84201593477144,
       "cc_by_type": {
        "choice": 54.747928387498305,
        "noul": -2.1582733812949617,
        "score": 27.936392798110976
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 26.84201593477144
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 67.20150275405432,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 21.180364705882337
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.20150275405432
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 15.402943169398917,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 45.11593333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 15.402943169398917
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 55.9409701754386,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 67.8229105263158
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 55.9409701754386
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 5.854853142458097,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -100,
        "score": 51.08411249999999
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 5.854853142458097
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 64.8121273057937,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 8.465999999999962
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 64.8121273057937
      },
      "ja": {
       "accuracy": 0.4,
       "cc": -0.4558023193010274,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": -40,
        "score": 24.41496000000002
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -7.808200529100527,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 21.01984285714287
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 95.45897222222222,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 86.37691666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 95.45897222222222
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 1.8647846753246806,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 31.56838000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 1.8647846753246806
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 30.657935079241657,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -19.999999999999996,
        "score": 34.82611162790696
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 30.657935079241657
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 41.932065296126616,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 19.999999999999996,
        "score": 48.224009677419346
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 41.932065296126616
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": -12.919919858156026,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60,
        "score": 23.3679
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": -2.1177119565217524,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": -62.87487500000004
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 66.71357365591398,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 64.65685
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 66.71357365591398
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 14.517088888888878,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -23.11540000000003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 14.517088888888878
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6626666666666666,
       "cc": 56.428557760385765,
       "cc_by_type": {
        "choice": 56.428557760385765
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 56.428557760385765
      },
      "noul": {
       "accuracy": 0.48,
       "cc": -4.0000000000000036,
       "cc_by_type": {
        "noul": -4.0000000000000036
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.504,
       "cc": 30.3285861010831,
       "cc_by_type": {
        "score": 30.3285861010831
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 30.3285861010831
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -20.217698039215694,
       "cc_by_type": {
        "choice": 52,
        "noul": -75,
        "score": -37.65309411764708
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": 6.881704004530096,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -11.111111111111116,
        "score": 6.290384615384637
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 6.881704004530096
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 30.13055826558266,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": 22.91200000000001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 30.13055826558266
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -35.508502631578956,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": -1.262350000000012
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 18.20034864130435,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 45.90539375
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 18.20034864130435
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -46.83123815513627,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100,
        "score": -52.443400000000004
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 12.310430690224806,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 37.771628205128195
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 12.310430690224806
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -14.343936656062217,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100,
        "score": 30.13892173913042
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.13333333333333333,
       "cc": -5.765200000000003,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -60,
        "score": 46.871066666666664
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 9.370813276231262,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 0,
        "score": 18.0482
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 9.370813276231262
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 35.32868872053872,
       "cc_by_type": {
        "choice": 100,
        "noul": -11.111111111111116,
        "score": 17.097177272727283
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 35.32868872053872
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -59.25189018817205,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -48.72341250000003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.4074074074074074,
       "cc": 20.06767288689841,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": 26.268277777777772
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 20.06767288689841
      },
      "model_routing": {
       "accuracy": 0.8110599078341014,
       "cc": 66.99794162946428,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 72.72727272727273,
        "score": 48.13668203124999
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.99794162946428
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 9.025348921756807,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 73.14130588235295
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 9.025348921756807
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 58.44408162393162,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 41.99891153846155
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 58.44408162393162
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 14.167175939341256,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50,
        "score": 7.279852941176479
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 14.167175939341256
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 8.994591932367145,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -66.66666666666667,
        "score": 37.12870333333333
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 8.994591932367145
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 71.28822061247423,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50,
        "score": 74.54081842105262
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 71.28822061247423
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.35230824724442,
      85.74985775934907
     ],
     "I": [
      27.368604889402985,
      34.51967568447579
     ],
     "capability": [
      54.162736400093735,
      58.62101341997018
     ],
     "composite": null,
     "seed": 34
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.31146464049283,
     "capability": 56.949321883834315,
     "intelligence": 30.5871791271758
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6626666666666666,
      "calibration": 76.12813377049179,
      "cc": 56.428557760385765,
      "cc_by_type": {
       "choice": 56.428557760385765
      },
      "cc_tiered": 58.807276377369945,
      "cc_tiered_open": 63.94109602078727,
      "cc_tiered_sealed": 53.67345673395262,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 56.428557760385765
     },
     "noul": {
      "accuracy": 0.48,
      "calibration": 87.72792117962464,
      "cc": -4.0000000000000036,
      "cc_by_type": {
       "noul": -4.0000000000000036
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.504,
      "calibration": 86.07833897136206,
      "cc": 30.3285861010831,
      "cc_by_type": {
       "score": 30.3285861010831
      },
      "cc_tiered": 32.95426100415746,
      "cc_tiered_open": 39.1002897500428,
      "cc_tiered_sealed": 26.808232258272128,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.3285861010831
     }
    },
    "raw_on_A": {
     "calibration": 76.8381791401458,
     "capability": 54.30955267637154,
     "intelligence": 31.780926212597272
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 19.283884056020195,
    "Emphasis on Accuracy 60:20:20": 15.138029371976163,
    "Emphasis on Cost 20:20:60": 21.408397429369167,
    "Emphasis on Speed 20:60:20": 23.364005403359833,
    "Intelligence only": 11.44664649202575
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Frontier Infra",
   "axes": {
    "calibration": 89.67673072100749,
    "cost": 68.53174752675467,
    "intelligence": 37.15024789587638,
    "speed": 90.97576909567059
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0545485704918033,
      "ece_score": 89.09028590163935,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 89.09028590163935,
      "score_before_time_limit": 89.09028590163935,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.4048257372654156,
      "brier": 0.12966919167437543,
      "ece": 0.02817181233243971,
      "n": 373,
      "score": 94.36563753351206,
      "score_before_time_limit": 94.36563753351206,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9408602150537635,
      "interval90_width": 0.4444444444444445,
      "n": 372,
      "nrps": 0.10121750178666497,
      "score": 85.57426872787106,
      "score_before_time_limit": 85.57426872787106,
      "time_limit_failed_share": 0,
      "top_ece": 0.09364856182795694
     }
    },
    "score": 89.67673072100749
   },
   "capability": 63.41348930844194,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 750,303 input + 1,485 output tokens over 1,485 decisions = 506.254545455 tokens/decision. GPU-time cost USD 0.0111928774598015/1,000; old list-price cost USD 0.0506754545454545/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0111928774598015/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05016869999999999
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011192877459801475
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jebadiah 9B v2.1 (Frontier Infra, Qwen3.5-9B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA GeForce RTX 5090 32 GB (sm_120, driver 595.71.05), Romania (EU) - different hardware from the RTX PRO 6000 the author measures on, so Speed is a lower bound on this card and not comparable to their 49.9 ms median. The author's OWN standalone server `jebadiah-serve --model <pinned local dir> --device cuda` from getainode/jebadiah @ 74a0cd92965529e90a411532e9887b544c931b2d (the commit behind the published jebadiah-27b row, independently code-reviewed PASS), installed from the author's own frozen uv.lock (`uv sync --frozen --no-dev --extra cuda`, torch 2.14.0+cu130, transformers 5.17.0, flash-linear-attention 0.5.2), all author defaults (dtype auto = bf16, batch-size 8, --max-prompt-tokens 8192, strict prompt-contract check ON, the repo's own temperatures.json, fp32 candidate logits), JEBADIAH_API_KEY unset. Isolation: server started inside a network namespace with loopback only (unshare -n), the cut proven before the sealed input was staged and re-proven immediately before the first sealed request, synthetic smoke gate passed in between; runner unchanged (run_v15.py, 0 resumes, rc 0); one use of input-selfhosted-1500 (under the pre-registration). Pod removed by exact id and verified gone four ways. Published Speed uses the standing x2 + 0.15 s self-hosted adjustment; raw p50 0.0502 s / p95 0.0837 s.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 45.02812038711811,
    "I_sealed": 29.272375404634655,
    "base": 37.15024789587638,
    "excess": 5.916218212127742,
    "gap": 15.755744982483453,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 65.08523959814883,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 60.36489462094998,
       "judge": 60.44825313117996,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 22.648033126294003,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 21.739130434782616,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 47.35108843691149,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.52056666666667,
       "hard": 53.42431944444446,
       "judge": 36.94557627118644,
       "standard": 49.22815555555554
      }
     },
     "sealed|choice": {
      "cc": 56.24785832875334,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 69.50444726810673,
       "hard": 49.52006476234531,
       "judge": 55.04407443682666,
       "standard": 64.88082682978275
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 37.93103448275863,
       "hard": -3.6363636363636376,
       "judge": -26.15384615384615,
       "standard": 25.373134328358216
      }
     },
     "sealed|score": {
      "cc": 31.56926788515063,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 54.8699552631579,
       "hard": 24.433701408450716,
       "judge": 25.61958750000002,
       "standard": 43.11457772727274
      }
     }
    }
   },
   "jevbench_score": 34.69183418142899,
   "key": "jebadiah-9b-v2-1",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0 (weights and base)",
   "listing": "ranked",
   "measured_in": "v1.6.25",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "frontier-infra/jebadiah-9b-v2-1@c5e49fab7ec8e9923e1f907d85911a53fb8e9c32",
   "noul_decisive": {
    "acc_among_decisive": 0.9279279279279279,
    "decisive": 222,
    "decisive_rate": 0.5951742627345844,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.67673072100749,
     "cost": 48.85607041331758,
     "intelligence": 37.15024789587638,
     "speed": 91.39972504887733
    },
    "board_release": "v1.6.43",
    "capability": 63.41348930844194,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 44,
    "capability_rank": 68,
    "composite_ci95": {
     "A": [
      21.591093369152567,
      38.23238527743113
     ],
     "B": [
      19.023355726025706,
      35.04565170961342
     ],
     "C": [
      14.993814839689282,
      26.55026755377161
     ]
    },
    "jevbench_score": 30.34536701293529,
    "measured_in": "v1.6.25",
    "method": "v1.6",
    "open_board_rank": 68,
    "rank": 100,
    "ranks": {
     "A": 100,
     "B": 101,
     "C": 105,
     "capability": 110
    },
    "scores": {
     "A": 30.34536701293529,
     "B": 27.339601213975246,
     "C": 21.073171536760626
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2362852499994915,
     "p50_s_raw": 0.04314262499974575,
     "p95_s_adjusted": 0.30661322509549793,
     "p95_s_raw": 0.07830661254774895
    },
    "views": {
     "Balanced 33:33:33": 27.110180335527737,
     "Emphasis on Accuracy 60:20:20": 23.49650090011741,
     "Emphasis on Cost 20:20:60": 26.54977055133271,
     "Emphasis on Speed 20:60:20": 32.85701676126051,
     "Intelligence only": 19.581330456126626
    }
   },
   "rank": 157,
   "ranked": true,
   "ranks": {
    "A": 157,
    "B": 158,
    "C": 159,
    "capability": 141
   },
   "repo": "https://huggingface.co/frontier-infra/jebadiah-9b-v2-1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 34.69183418142899,
    "B": 30.476670404268415,
    "C": 24.09155151488125
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.25061776800284863,
    "p50_s_raw": 0.0503088840014243,
    "p95_s_adjusted": 0.3187214060009864,
    "p95_s_raw": 0.08436070300049323
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7272727272727273,
       "cc": 45.461111111111116,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100,
        "score": -21.94999999999998
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 45.461111111111116
      },
      "calibration_gen": {
       "accuracy": 0.39603960396039606,
       "cc": 8.965266960498408,
       "cc_by_type": {
        "choice": 23.647932131495228,
        "noul": -4.0000000000000036,
        "score": 7.247868749999997
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 8.965266960498408
      },
      "composite_gen": {
       "accuracy": 0.5742574257425742,
       "cc": 25.502561765455255,
       "cc_by_type": {
        "choice": 64.22764227642277,
        "noul": -18.518518518518523,
        "score": 30.79856153846151
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 25.502561765455255
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 79.02633518518519,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 77.07900555555555
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 79.02633518518519
      },
      "jaggedness": {
       "accuracy": 0.42777777777777776,
       "cc": 7.105247243920597,
       "cc_by_type": {
        "choice": 32.14629451395574,
        "noul": -35.13513513513513,
        "score": 24.304582352941186
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 7.105247243920597
      },
      "llm_router": {
       "accuracy": 0.7955271565495208,
       "cc": 68.52774777747052,
       "cc_by_type": {
        "choice": 70.67887646385489,
        "noul": 71.13402061855669,
        "score": 63.770346249999996
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.52774777747052
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -21.368183456062837,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -100,
        "score": -14.620014285714289
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6145454545454545,
       "cc": 30.838639188636876,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": -16.666666666666664,
        "score": 43.41799342723004
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 30.838639188636876
      },
      "negation_pair_gen": {
       "accuracy": 0.5116279069767442,
       "cc": 10.267682222222229,
       "cc_by_type": {
        "choice": 59.99999999999999,
        "noul": -33.333333333333336,
        "score": 4.136380000000028
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 10.267682222222229
      },
      "option_order": {
       "accuracy": 0.8958333333333334,
       "cc": 86.68442077230361,
       "cc_by_type": {
        "choice": 86.68442077230361
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 86.68442077230361
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 70.84381691306692,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 80,
        "score": 62.56148076923078
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 70.84381691306692
      },
      "usecase": {
       "accuracy": 0.55,
       "cc": 27.858113095424844,
       "cc_by_type": {
        "choice": 52.205882352941195,
        "noul": -4.1666666666666625,
        "score": 35.535123600000006
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 27.858113095424844
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 4.7372142857142805,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0,
        "score": -28.645500000000013
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 4.7372142857142805
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 39.28832631848439,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": 26.006300000000017
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 39.28832631848439
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 70.35387155316147,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75,
        "score": 67.92002173913043
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 70.35387155316147
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 21.789848975791433,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -100,
        "score": 98.8891
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.789848975791433
      },
      "en": {
       "accuracy": 0.6072980017376195,
       "cc": 35.34340890177539,
       "cc_by_type": {
        "choice": 56.28562014132117,
        "noul": 14.388489208633093,
        "score": 35.35611735537191
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 35.34340890177539
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 66.55888314621119,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 19.252505882352942
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.55888314621119
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 29.99688269581056,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 56.110866666666666
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 29.99688269581056
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 55.07443787409701,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 86.39978421052632
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 55.07443787409701
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 3.0043369878957136,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -50,
        "score": -24.227212500000018
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 3.0043369878957136
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 61.87361175444456,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 13.680071428571395
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 61.87361175444456
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 25.42277845379931,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 0,
        "score": 33.45658
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 25.42277845379931
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -7.526621957671959,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 21.864578571428574
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.65298333333334,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 96.62561666666667
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.65298333333334
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -0.26805731601731964,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 51.14387999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 32.30207713293114,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 32.141102325581386
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 32.30207713293114
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 45.18160400580404,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 19.999999999999996,
        "score": 57.97262580645162
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 45.18160400580404
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": -0.5227982269503476,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 35.02735000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 10.888600543478256,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50,
        "score": -73.85593750000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 10.888600543478256
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 69.77072365591397,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 73.82829999999998
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 69.77072365591397
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 13.044855555555557,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -27.532099999999993
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 13.044855555555557
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6813333333333333,
       "cc": 58.83962571040395,
       "cc_by_type": {
        "choice": 58.83962571040395
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 58.83962571040395
      },
      "noul": {
       "accuracy": 0.5493333333333333,
       "cc": 9.866666666666667,
       "cc_by_type": {
        "noul": 9.866666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 9.866666666666667
      },
      "score": {
       "accuracy": 0.544,
       "cc": 36.35312382671484,
       "cc_by_type": {
        "score": 36.35312382671484
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.35312382671484
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": -11.250268627450984,
       "cc_by_type": {
        "choice": 68,
        "noul": -50,
        "score": -51.75080588235295
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 16.864947946417512,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 11.111111111111116,
        "score": -4.6156461538461535
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 16.864947946417512
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 36.58365826558265,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": 42.27129999999999
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 36.58365826558265
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -25.61673798732944,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100,
        "score": 16.717037499999996
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 16.98094864130434,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 42.24719374999998
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 16.98094864130434
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -50.77358270440254,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -100,
        "score": -76.84905000000009
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 14.320775486604896,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -71.42857142857143,
        "score": 33.998741025641024
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 14.320775486604896
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -9.09330015654193,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100,
        "score": 35.437869565217404
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.2,
       "cc": -0.8783444444444442,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60,
        "score": 40.698299999999996
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 33.657132663898075,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 33.33333333333333,
        "score": 44.72585909090909
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 33.657132663898075
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 32.8623636676846,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 21.326109090909096
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 32.8623636676846
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -54.82240618279571,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100,
        "score": -67.69302500000003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 22.22430251652804,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": 32.73816666666666
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 22.22430251652804
      },
      "model_routing": {
       "accuracy": 0.8248847926267281,
       "cc": 69.68780904581529,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 75.75757575757575,
        "score": 53.17598124999998
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 69.68780904581529
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 15.739794068721551,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 70.25201176470588
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 15.739794068721551
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 56.08058175249388,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 51.62150384615387
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 56.08058175249388
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 22.892261682604083,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": -1.7665647058823453
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 22.892261682604083
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 3.1867240579710128,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -66.66666666666667,
        "score": 34.19785333333333
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 3.1867240579710128
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 76.21831885808827,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50,
        "score": 89.33111315789473
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 76.21831885808827
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.9065626572593,
      89.19278572343606
     ],
     "I": [
      32.34737884360564,
      41.02255692898279
     ],
     "capability": [
      58.747843004374815,
      64.1123155056803
     ],
     "composite": null,
     "seed": 35
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.67673072100749,
     "capability": 63.41348930844194,
     "intelligence": 37.15024789587638
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6813333333333333,
      "calibration": 89.09028590163935,
      "cc": 58.83962571040395,
      "cc_by_type": {
       "choice": 58.83962571040395
      },
      "cc_tiered": 60.666548963451085,
      "cc_tiered_open": 65.08523959814883,
      "cc_tiered_sealed": 56.24785832875334,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 58.83962571040395
     },
     "noul": {
      "accuracy": 0.5493333333333333,
      "calibration": 94.36563753351206,
      "cc": 9.866666666666667,
      "cc_by_type": {
       "noul": 9.866666666666667
      },
      "cc_tiered": 11.324016563147001,
      "cc_tiered_open": 22.648033126294003,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 9.866666666666667
     },
     "score": {
      "accuracy": 0.544,
      "calibration": 85.57426872787106,
      "cc": 36.35312382671484,
      "cc_by_type": {
       "score": 36.35312382671484
      },
      "cc_tiered": 39.46017816103106,
      "cc_tiered_open": 47.35108843691149,
      "cc_tiered_sealed": 31.56926788515063,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.35312382671484
     }
    },
    "raw_on_A": {
     "calibration": 83.97311382409126,
     "capability": 61.0776332772807,
     "intelligence": 38.182152730470136
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 31.545215323752238,
    "Emphasis on Accuracy 60:20:20": 25.957898155854668,
    "Emphasis on Cost 20:20:60": 33.79178740327558,
    "Emphasis on Speed 20:60:20": 37.05806924043796,
    "Intelligence only": 20.509030904751533
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jeetu456 (Hugging Face)",
   "axes": {
    "calibration": 73.13553830274921,
    "cost": 85.8779611353149,
    "intelligence": 10.191329118370533,
    "speed": 90.89924129769977
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.167373672238061,
      "ece_score": 66.52526555238781,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 66.52526555238781,
      "score_before_time_limit": 66.52526555238781,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.6166219839142091,
      "brier": 0.24547394934030595,
      "ece": 0.14238080114387147,
      "n": 373,
      "score": 71.52383977122571,
      "score_before_time_limit": 71.52383977122571,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.956989247311828,
      "interval90_width": 0.8436379928315415,
      "n": 372,
      "nrps": 0.1666959236117213,
      "score": 81.3575095846341,
      "score_before_time_limit": 81.3575095846341,
      "time_limit_failed_share": 0,
      "top_ece": 0.10307694234779832
     }
    },
    "score": 73.13553830274921
   },
   "capability": 41.66343371055987,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 726,138 input + 0 output tokens over 1,485 decisions = 488.981818182 tokens/decision. GPU-time cost USD 0.00295620555555556/1,000; old list-price cost USD 0.00977963636363636/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00295620555555556/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.00968184
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0029562055555555564
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jeetu Decider 2B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-25",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.2",
    "results_sha256": "ac51a64e34c6e2dc1042f3e2f8f3da758d12de0e1e07e5c694cad9f72e3ecc0d",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Author's native server on an evaluator-rented Verda NVIDIA RTX A6000 48 GB, FIN-02 Finland, EU, PAYG non-spot, measured 25 October 2026 about 06:57-07:11 UTC. Pinned local checkpoint and serving code, serial loopback requests, network-isolated read-only container, offline flags and empty provider credentials. Snapshot hashes and threshold verified before held-out input. All 1500 requests completed; node shred and exact-instance teardown verified by the measurement receipts. Approximately USD 0.20 rental is separate from the estimated inference tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 13.480221157458207,
    "I_sealed": 6.90243707928286,
    "base": 10.191329118370533,
    "excess": -3.2617426921803645,
    "gap": 6.5777840781753465,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 28.503847243644508,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 1.8559295375904445,
       "judge": 40.672379696769944,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -73.91304347826086,
       "judge": -60,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 11.936816228730116,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.1728879503081,
       "hard": -4.831012511061372,
       "judge": 14.832332644685952,
       "standard": 19.011163223590355
      }
     },
     "sealed|choice": {
      "cc": 20.707311237848582,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 23.76111817026684,
       "hard": 13.091245518676985,
       "judge": 25.07345739471108,
       "standard": 27.8633199746889
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -10.344827586206895,
       "hard": -58.18181818181818,
       "judge": -50.76923076923077,
       "standard": -46.26865671641791
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 13.144316058948258,
       "hard": -10.402810416749485,
       "judge": 1.0377813440464623,
       "standard": 4.383223925828306
      }
     }
    }
   },
   "jevbench_score": 1.23608490981135,
   "key": "jeetu-decider-2b",
   "last_measured_on": "2026-10-25",
   "licence": "Apache-2.0 per the model card and LICENSE (LICENSE sha256 prefix 50cbab8a); public, ungated",
   "listing": "ranked",
   "measured_in": "v1.7.2",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Jeetu456/decider-2b@cd0216309bf590f387500d78d1970af15d04baf5",
   "new_in": "v1.7.2",
   "noul_decisive": {
    "acc_among_decisive": 0.7552447552447552,
    "decisive": 143,
    "decisive_rate": 0.38337801608579086,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 238,
   "ranked": true,
   "ranks": {
    "A": 238,
    "B": 238,
    "C": 238,
    "capability": 244
   },
   "repo": "https://huggingface.co/Jeetu456/decider-2b",
   "scores": {
    "A": 1.23608490981135,
    "B": 0.8932004460686231,
    "C": 0.858392298480104
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27145866199989543,
    "p50_s_raw": 0.060729330999947706,
    "p95_s_adjusted": 0.2994829938999373,
    "p95_s_raw": 0.07474149694996866
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "First measurement of Jeetu456/decider-2b at cd0216309bf590f387500d78d1970af15d04baf5, a Qwen3.5-2B-family fine-tune with a BF16 backbone and FP32 option-pointer decision head. The author's native SystemOne server ran unmodified, with native choice, noul and score probabilities and no generated explanation. The author default abstain threshold 0.55 was frozen before measurement. No answered choice request matched the native English IDK detector, so its choice-only threshold redistribution did not activate. Separately, the scorer records 230 of 373 answered Noul items within its 0.20-0.80 abstention band; this is not attributed to the server's choice threshold. The author documents a threshold-0 alternative, which was not measured. Optional causal-conv1d was omitted in favour of the author's supported F.conv1d fallback, the same maths up to BF16 reassociation; optional-kernel equivalence was not measured. The two weight shards and pointer head share no recorded LFS hashes with 28 compared repositories, including Mapika/decider-2b and Qwen/Qwen3.5-2B. Tokenizer sharing with the base does not imply checkpoint sharing. The model card warns that some When2Call test questions may have been seen during training; training-data contamination against JevBench was not checked. Exactly 1485 answers and 15 preregistered native HTTP 413 over-length refusals; refusals count wrong in the 1500-item denominator, with no evaluator truncation. First 50 responses repeated bit-identically. Earlier provider and installer failures stopped before held-out input and produced no measurement; only the successful recovery attempt is scored.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.36363636363636365,
       "cc": -5.26566620521879,
       "cc_by_type": {
        "choice": 8.333333333333334,
        "noul": 0,
        "score": -24.130331948989703
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.36633663366336633,
       "cc": -18.09507631650604,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": -76,
        "score": -11.47716956457645
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.3465346534653465,
       "cc": -12.53634745058568,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": -62.96296296296296,
        "score": 6.654733619336007
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.3541666666666667,
       "cc": -5.367242269769188,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -60,
        "score": 10.564939857359102
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.21666666666666667,
       "cc": -24.32684284805823,
       "cc_by_type": {
        "choice": 0.3849855630414015,
        "noul": -78.37837837837837,
        "score": 5.012864271162276
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.5718849840255591,
       "cc": 30.373888981985846,
       "cc_by_type": {
        "choice": 46.112529717354946,
        "noul": 34.020618556701024,
        "score": 10.98851867190158
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 30.373888981985846
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -22.468892650142067,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -100,
        "score": 31.56239421452226
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.3236363636363636,
       "cc": -14.907719401151889,
       "cc_by_type": {
        "choice": 15.878708835996095,
        "noul": -61.111111111111114,
        "score": 0.5092440716593538
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.22093023255813954,
       "cc": -32.94249583313457,
       "cc_by_type": {
        "choice": 10.000000000000002,
        "noul": -100,
        "score": -8.82748749940372
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.2916666666666667,
       "cc": 9.454061251664452,
       "cc_by_type": {
        "choice": 9.454061251664452
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 9.454061251664452
      },
      "perturb_pair": {
       "accuracy": 0.5,
       "cc": 27.011988530271406,
       "cc_by_type": {
        "choice": 33.933933933933936,
        "noul": 39.99999999999999,
        "score": 7.102031656880293
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 27.011988530271406
      },
      "usecase": {
       "accuracy": 0.29545454545454547,
       "cc": -18.975882010691276,
       "cc_by_type": {
        "choice": 21.568627450980426,
        "noul": -79.16666666666666,
        "score": 0.6703931836123944
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.1875,
       "cc": -27.028502932338963,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": -50,
        "score": -16.7997945113026
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": -31.578947368421055,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.38461538461538464,
       "cc": 5.327048526171981,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -33.333333333333336,
        "score": 11.526460478669554
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 5.327048526171981
      },
      "de": {
       "accuracy": 0.5,
       "cc": 22.585572825556824,
       "cc_by_type": {
        "choice": 36.283185840707965,
        "noul": 0,
        "score": 31.4735326359625
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 22.585572825556824
      },
      "el": {
       "accuracy": 0.4,
       "cc": 17.75915975282979,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 86.98325207040745
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 17.75915975282979
      },
      "en": {
       "accuracy": 0.35621198957428324,
       "cc": -6.471407637796111,
       "cc_by_type": {
        "choice": 21.577720555033455,
        "noul": -41.72661870503597,
        "score": 0.7346752366141795
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.5161290322580645,
       "cc": 6.1030798418069665,
       "cc_by_type": {
        "choice": 41.27243066884176,
        "noul": -25,
        "score": 2.036808856579142
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 6.1030798418069665
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -11.672961722740494,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": -3.3248960971832275
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.2857142857142857,
       "cc": 7.162351721318868,
       "cc_by_type": {
        "choice": 15.294117647058824,
        "noul": 0,
        "score": 6.1929375168977785
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 7.162351721318868
      },
      "hi": {
       "accuracy": 0.23809523809523808,
       "cc": -33.07011040076162,
       "cc_by_type": {
        "choice": -0.5586592178770929,
        "noul": -100,
        "score": 1.3483280155922373
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.6666666666666666,
       "cc": 41.67606794164006,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 33.33333333333333,
        "score": 19.75410665682459
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 41.67606794164006
      },
      "ja": {
       "accuracy": 0.32,
       "cc": -14.239244140167964,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": -40,
        "score": -16.935365462600792
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -25.743616251397686,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": 11.658040134695824
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.35714285714285715,
       "cc": -7.837299247044363,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -33.333333333333336,
        "score": -23.51189774113309
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.1,
       "cc": -36.246220908316495,
       "cc_by_type": {
        "choice": -29.87012987012987,
        "noul": -100,
        "score": 21.131467145180395
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -39.516129032258064,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 19.86833017287262,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -19.999999999999996,
        "score": 10.07473237219384
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 19.86833017287262
      },
      "pt": {
       "accuracy": 0.35714285714285715,
       "cc": -17.055431038623208,
       "cc_by_type": {
        "choice": 15.144372421921032,
        "noul": -60,
        "score": -6.3106655377906495
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -22.675016636258533,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": -60,
        "score": 19.634524559309497
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.38461538461538464,
       "cc": -11.285191632508294,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -50,
        "score": 3.1009468416055563
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.14285714285714285,
       "cc": -41.48530551019099,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": -100,
        "score": 4.576341533943151
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.3,
       "cc": -42.12495739561377,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": -33.333333333333336,
        "score": -91.3466235992707
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.3973333333333333,
       "cc": 22.156949042270238,
       "cc_by_type": {
        "choice": 22.156949042270238
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 22.156949042270238
      },
      "noul": {
       "accuracy": 0.288,
       "cc": -42.400000000000006,
       "cc_by_type": {
        "noul": -42.400000000000006
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.3626666666666667,
       "cc": 1.5247135367799758,
       "cc_by_type": {
        "score": 1.5247135367799758
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 1.5247135367799758
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.18181818181818182,
       "cc": -37.19244222486281,
       "cc_by_type": {
        "choice": 4,
        "noul": -100,
        "score": -15.57732667458842
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": -5.250487583046673,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -77.77777777777779,
        "score": 17.926936146650196
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.14285714285714285,
       "cc": -24.065329218259038,
       "cc_by_type": {
        "choice": -13.821138211382115,
        "noul": -50,
        "score": -8.374849443394993
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.11764705882352941,
       "cc": -42.08651989424725,
       "cc_by_type": {
        "choice": -16.959064327485383,
        "noul": -100,
        "score": -9.300495355256388
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": -29.15008889620631,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": -0.4937449494884838
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -40.404097689041585,
       "cc_by_type": {
        "choice": -0.62893081761006,
        "noul": -100,
        "score": -20.583362249514693
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.36666666666666664,
       "cc": -7.38985968555667,
       "cc_by_type": {
        "choice": 41.17647058823529,
        "noul": -71.42857142857143,
        "score": 8.082521783666131
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.13636363636363635,
       "cc": -33.62558431586917,
       "cc_by_type": {
        "choice": 5.9233449477351945,
        "noul": -100,
        "score": -6.800097895342705
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.13333333333333333,
       "cc": -18.50616864004334,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -100,
        "score": 27.814827413203304
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.09090909090909091,
       "cc": -42.99198796979619,
       "cc_by_type": {
        "choice": -15.631691648822272,
        "noul": -100,
        "score": -13.344272260566292
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.28,
       "cc": -17.175747850162406,
       "cc_by_type": {
        "choice": 41.860465116279066,
        "noul": -77.77777777777779,
        "score": -15.6099308889885
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -37.47063532890442,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -47.895776954455194
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.4444444444444444,
       "cc": 7.227009572977027,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": -12.253712163986386
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 7.227009572977027
      },
      "model_routing": {
       "accuracy": 0.5990783410138248,
       "cc": 33.679445764993076,
       "cc_by_type": {
        "choice": 55.584415584415595,
        "noul": 27.27272727272727,
        "score": 18.181194437836368
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 33.679445764993076
      },
      "moderation_trust_safety": {
       "accuracy": 0.16666666666666666,
       "cc": -37.503462925284396,
       "cc_by_type": {
        "choice": 7.869481765834927,
        "noul": -100,
        "score": -20.379870541688106
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.4444444444444444,
       "cc": -17.763511079407877,
       "cc_by_type": {
        "choice": 33.147632311977716,
        "noul": -100,
        "score": 13.56183444979865
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -12.460163029339085,
       "cc_by_type": {
        "choice": -3.4482758620689666,
        "noul": -50,
        "score": 16.067786774051708
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.34782608695652173,
       "cc": -2.1721758116981107,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -33.333333333333336,
        "score": 28.26608126055784
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.4166666666666667,
       "cc": 10.715951935922815,
       "cc_by_type": {
        "choice": 14.590747330960852,
        "noul": 0,
        "score": 17.557108476807592
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 10.715951935922815
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.41730934673446,
      76.54064416197126
     ],
     "I": [
      7.7289816011201555,
      12.711138187380783
     ],
     "capability": [
      38.189515300908745,
      43.93129718310302
     ],
     "composite": null,
     "seed": 36
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.13553830274921,
     "capability": 41.66343371055987,
     "intelligence": 10.191329118370533
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3973333333333333,
      "calibration": 66.52526555238781,
      "cc": 22.156949042270238,
      "cc_by_type": {
       "choice": 22.156949042270238
      },
      "cc_tiered": 24.605579240746543,
      "cc_tiered_open": 28.503847243644508,
      "cc_tiered_sealed": 20.707311237848582,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 22.156949042270238
     },
     "noul": {
      "accuracy": 0.288,
      "calibration": 71.52383977122571,
      "cc": -42.400000000000006,
      "cc_by_type": {
       "noul": -42.400000000000006
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.3626666666666667,
      "calibration": 81.3575095846341,
      "cc": 1.5247135367799758,
      "cc_by_type": {
       "score": 1.5247135367799758
      },
      "cc_tiered": 5.968408114365058,
      "cc_tiered_open": 11.936816228730116,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 1.5247135367799758
     }
    },
    "raw_on_A": {
     "calibration": 69.59366240375074,
     "capability": 40.02042090976124,
     "intelligence": 10.447179415771737
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 1.0320247712171036,
    "Emphasis on Accuracy 60:20:20": 0.6552603375303412,
    "Emphasis on Cost 20:20:60": 1.4419723763109205,
    "Emphasis on Speed 20:60:20": 1.454965707681833,
    "Intelligence only": 0.423401577764023
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": null,
   "axes": {
    "calibration": 91.10757097649757,
    "cost": 54.523640029233874,
    "intelligence": 71.89886397161271,
    "speed": 88.47496650745126
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05187038243550904,
      "ece_score": 89.6259235128982,
      "mean_tvd": null,
      "n": 735,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 89.6259235128982,
      "score_before_time_limit": 89.6259235128982,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.07263564703720832,
      "ece": 0.03721686055451093,
      "n": 373,
      "score": 92.55662788909783,
      "score_before_time_limit": 92.55662788909783,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9703504043126685,
      "interval90_width": 0.29222821203953275,
      "n": 371,
      "nrps": 0.06110092445996998,
      "score": 91.1401615274967,
      "score_before_time_limit": 91.1401615274967,
      "time_limit_failed_share": 0,
      "top_ece": 0.0580479224950481
     }
    },
    "score": 91.10757097649757
   },
   "capability": 81.50321747405513,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 810,973 input + 0 output tokens over 1,479 decisions = 548.325219743 tokens/decision. GPU-time cost USD 0.0328003641810933/1,000; old list-price cost USD 0.0493492697768763/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0328003641810933/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.048658379999999994
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03280036418109335
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jeff-1.0-Large",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-09",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 73.94592822865295,
    "I_sealed": 69.85179971457248,
    "base": 71.89886397161271,
    "excess": -2.993969230169018,
    "gap": 4.094128514080467,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 76.92440600606106,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396664,
       "judge": 68.35860250494397,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 85.29951690821257,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 70,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 59.61386177168524,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.31995405652513,
       "hard": 69.76831247393966,
       "judge": 63.52914643184884,
       "standard": 27.578987234511022
      }
     },
     "sealed|choice": {
      "cc": 81.31363869979835,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 88.88774565284494,
       "hard": 78.22172200337647,
       "judge": 79.04328018223235,
       "standard": 87.11595639246778
      }
     },
     "sealed|noul": {
      "cc": 77.2270301416456,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 87.5,
       "hard": 70.6896551724138,
       "judge": 73.13432835820895,
       "standard": 91.30434782608697
      }
     },
     "sealed|score": {
      "cc": 51.01473030227352,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 79.67608755640215,
       "hard": 44.71435253600092,
       "judge": 42.59576964785805,
       "standard": 61.913248189377576
      }
     }
    }
   },
   "jevbench_score": 73.35712724451902,
   "key": "jeff_1_0_large",
   "last_measured_on": "2026-10-09",
   "listing": "ranked",
   "measured_in": "v1.6.2",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "275325a5eab52cce4f109a09ee3832f2eb89fe2f",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.10757097649757,
     "cost": 49.20157807658991,
     "intelligence": 71.89886397161271,
     "speed": 88.82411852274376
    },
    "board_release": "v1.6.43",
    "capability": 81.50321747405513,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 5,
    "capability_rank": 9,
    "composite_ci95": {
     "A": [
      67.07482591953416,
      69.37460423313809
     ],
     "B": [
      66.69110907316322,
      70.03338191380395
     ],
     "C": [
      67.07482591953416,
      69.37460423313809
     ]
    },
    "jevbench_score": 68.59329511867139,
    "measured_in": "v1.6.2",
    "method": "v1.6",
    "open_board_rank": 15,
    "rank": 32,
    "ranks": {
     "A": 32,
     "B": 25,
     "C": 28,
     "capability": 20
    },
    "scores": {
     "A": 68.59329511867139,
     "B": 68.79639624966302,
     "C": 68.59329511867139
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3101211428642273,
     "p50_s_raw": 0.08006057143211365,
     "p95_s_adjusted": 0.4227238700725138,
     "p95_s_raw": 0.1363619350362569
    },
    "views": {
     "Balanced 33:33:33": 63.85752832522542,
     "Emphasis on Accuracy 60:20:20": 66.04447649061674,
     "Emphasis on Cost 20:20:60": 56.205881793219106,
     "Emphasis on Speed 20:60:20": 71.19191780408633,
     "Intelligence only": 69.62097236625647
    }
   },
   "rank": 42,
   "ranked": true,
   "ranks": {
    "A": 42,
    "B": 29,
    "C": 35,
    "capability": 25
   },
   "repo": "https://huggingface.co/jgeuter/Jeff-1.0-Large",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.35712724451902,
    "B": 73.06076143508588,
    "C": 73.35712724451902
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.31703930497169497,
    "p50_s_raw": 0.08351965248584747,
    "p95_s_adjusted": 0.4481157846748829,
    "p95_s_raw": 0.14905789233744146
   },
   "status": {
    "answered_ok": 1479,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.96,
       "cc": 96.9377101225352,
       "cc_by_type": {
        "choice": 91.1111111111111,
        "noul": 100,
        "score": 99.70201925649454
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 96.9377101225352
      },
      "calibration_gen": {
       "accuracy": 0.78125,
       "cc": 57.561533071029515,
       "cc_by_type": {
        "choice": 75.30864197530865,
        "noul": 66.66666666666667,
        "score": 30.709290571113236
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 57.561533071029515
      },
      "composite_gen": {
       "accuracy": 0.92,
       "cc": 79.81401737255631,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 60.00000000000001,
        "score": 82.69408463799414
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 79.81401737255631
      },
      "fairness": {
       "accuracy": 0.9285714285714286,
       "cc": 91.50025446299001,
       "cc_by_type": {
        "choice": 79.66101694915254,
        "noul": 100,
        "score": 94.8397464398175
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 91.50025446299001
      },
      "jaggedness": {
       "accuracy": 0.6604938271604939,
       "cc": 47.147992393452604,
       "cc_by_type": {
        "choice": 64.34761079475119,
        "noul": 50,
        "score": 27.09636638560662
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 47.147992393452604
      },
      "llm_router": {
       "accuracy": 0.9318181818181818,
       "cc": 85.66561133970038,
       "cc_by_type": {
        "choice": 91.97360206902702,
        "noul": 97.91666666666667,
        "score": 67.10656528340743
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 85.66561133970038
      },
      "long_state": {
       "accuracy": 0,
       "cc": -131.99233716475095,
       "cc_by_type": {
        "choice": -29.310344827586214,
        "noul": -100,
        "score": -266.66666666666663
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.8529411764705882,
       "cc": 77.55088834993914,
       "cc_by_type": {
        "choice": 89.47368421052633,
        "noul": 75,
        "score": 68.1789808392911
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 77.55088834993914
      },
      "multilingual": {
       "accuracy": 0.8350877192982457,
       "cc": 75.39071926905552,
       "cc_by_type": {
        "choice": 79.45205479452055,
        "noul": 78.04878048780488,
        "score": 68.67132252484116
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 75.39071926905552
      },
      "negation_pair_gen": {
       "accuracy": 0.9183673469387755,
       "cc": 82.11185908219272,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 94.11764705882352,
        "score": 58.88459685442133
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 82.11185908219272
      },
      "option_order": {
       "accuracy": 0.9365079365079365,
       "cc": 91.91102123356926,
       "cc_by_type": {
        "choice": 91.91102123356926
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 91.91102123356926
      },
      "perturb_pair": {
       "accuracy": 0.9032258064516129,
       "cc": 82.33336907809512,
       "cc_by_type": {
        "choice": 87.42138364779876,
        "noul": 83.33333333333333,
        "score": 76.24539025315326
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 82.33336907809512
      },
      "usecase": {
       "accuracy": 0.7549019607843137,
       "cc": 66.3546834985278,
       "cc_by_type": {
        "choice": 76.10156833457805,
        "noul": 76,
        "score": 46.96248216100536
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 66.3546834985278
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8235294117647058,
       "cc": 62.110383554231476,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": 5.378769710313469
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 62.110383554231476
      },
      "cs": {
       "accuracy": 0.6666666666666666,
       "cc": 45.97701149425287,
       "cc_by_type": {
        "choice": 58.62068965517241,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 45.97701149425287
      },
      "da": {
       "accuracy": 0.75,
       "cc": 55.709291217777256,
       "cc_by_type": {
        "choice": 68.88504753673294,
        "noul": 50,
        "score": 48.24282611659883
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 55.709291217777256
      },
      "de": {
       "accuracy": 0.95,
       "cc": 91.29312298128339,
       "cc_by_type": {
        "choice": 92.58954302181968,
        "noul": 100,
        "score": 81.28982592203047
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 91.29312298128339
      },
      "el": {
       "accuracy": 0.7647058823529411,
       "cc": 58.66399707665688,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 19.999999999999996,
        "score": 94.08722932520874
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 58.66399707665688
      },
      "en": {
       "accuracy": 0.8290748898678414,
       "cc": 71.06721855084093,
       "cc_by_type": {
        "choice": 80.78832406671961,
        "noul": 79.8507462686567,
        "score": 52.562585317146485
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 71.06721855084093
      },
      "es": {
       "accuracy": 0.9333333333333333,
       "cc": 88.55170639268187,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100,
        "score": 65.65511917804558
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 88.55170639268187
      },
      "fi": {
       "accuracy": 1,
       "cc": 93.91862104229479,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 81.75586312688439
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 93.91862104229479
      },
      "fr": {
       "accuracy": 1,
       "cc": 98.41120186775261,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.23360560325787
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 98.41120186775261
      },
      "hi": {
       "accuracy": 0.8846153846153846,
       "cc": 84.11210683870443,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 52.3363205161133
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 84.11210683870443
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.8461538461538461,
       "cc": 79.18660535597223,
       "cc_by_type": {
        "choice": 83.98576512455516,
        "noul": 100,
        "score": 53.57405094336154
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 79.18660535597223
      },
      "ja": {
       "accuracy": 0.6818181818181818,
       "cc": 48.67304788730957,
       "cc_by_type": {
        "choice": 71.58642462509866,
        "noul": 75,
        "score": -0.567280963169936
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 48.67304788730957
      },
      "ko": {
       "accuracy": 0.8888888888888888,
       "cc": 90.81791628019677,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 100,
        "score": 99.12041550725698
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 90.81791628019677
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.4882036536204,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.4646109608612
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 99.4882036536204
      },
      "nl": {
       "accuracy": 0.8823529411764706,
       "cc": 76.76055951504456,
       "cc_by_type": {
        "choice": 85.62300319488817,
        "noul": 100,
        "score": 44.65867535024552
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 76.76055951504456
      },
      "no": {
       "accuracy": 0.75,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 64.15309188662947,
       "cc_by_type": {
        "choice": 92.46231155778895,
        "noul": 50,
        "score": 49.99696410209948
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 64.15309188662947
      },
      "pt": {
       "accuracy": 0.7916666666666666,
       "cc": 67.0299432480883,
       "cc_by_type": {
        "choice": 61.918194640338506,
        "noul": 75,
        "score": 64.17163510392639
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 67.0299432480883
      },
      "sv": {
       "accuracy": 0.5,
       "cc": 34.30530473285898,
       "cc_by_type": {
        "choice": 37.8238341968912,
        "noul": 33.33333333333333,
        "score": 31.758746668352412
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 34.30530473285898
      },
      "tr": {
       "accuracy": 0.8181818181818182,
       "cc": 76.84926416089685,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 52.28692291747318
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 76.84926416089685
      },
      "uk": {
       "accuracy": 0.6666666666666666,
       "cc": 70.01912003849547,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 74.57348914774444
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 70.01912003849547
      },
      "zh": {
       "accuracy": 0.9166666666666666,
       "cc": 77.21435846743238,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 31.64307540229713
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 77.21435846743238
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8546666666666667,
       "cc": 81.18490956865303,
       "cc_by_type": {
        "choice": 81.18490956865303
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.18490956865303
      },
      "noul": {
       "accuracy": 0.9013333333333333,
       "cc": 80.26666666666667,
       "cc_by_type": {
        "noul": 80.26666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 80.26666666666667
      },
      "score": {
       "accuracy": 0.7226666666666667,
       "cc": 55.788639506853855,
       "cc_by_type": {
        "score": 55.788639506853855
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 55.788639506853855
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6666666666666666,
       "cc": 49.563854302825206,
       "cc_by_type": {
        "choice": 57.597173144876315,
        "noul": 50,
        "score": 41.0943897635993
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 49.563854302825206
      },
      "code_linting": {
       "accuracy": 0.8181818181818182,
       "cc": 84.20240094634025,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 52.60720283902074
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 84.20240094634025
      },
      "customer_support": {
       "accuracy": 0.7692307692307693,
       "cc": 43.650863010018696,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100,
        "score": -25.5691501003787
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 43.650863010018696
      },
      "demand_forecasting": {
       "accuracy": 0.38095238095238093,
       "cc": 19.258228775832507,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": 50,
        "score": -14.303235750424559
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 19.258228775832507
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8571428571428571,
       "cc": 75.10276024397294,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 65.30828073191881
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 75.10276024397294
      },
      "feature_extraction": {
       "accuracy": 0.8125,
       "cc": 71.104810968882,
       "cc_by_type": {
        "choice": 68.75,
        "noul": 100,
        "score": 44.56443290664603
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 71.104810968882
      },
      "financial_crime": {
       "accuracy": 0.7142857142857143,
       "cc": 74.39743035918184,
       "cc_by_type": {
        "choice": 89.36170212765957,
        "noul": 100,
        "score": 33.83058894988594
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 74.39743035918184
      },
      "gaming": {
       "accuracy": 0.7058823529411765,
       "cc": 43.987755987145654,
       "cc_by_type": {
        "choice": 57.84543325526932,
        "noul": 0,
        "score": 74.11783470616764
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 43.987755987145654
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 68.17072508006011,
       "cc_by_type": {
        "choice": 68.42105263157895,
        "noul": 100,
        "score": 36.09112260860137
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 68.17072508006011
      },
      "knowledge_graphs": {
       "accuracy": 0.8095238095238095,
       "cc": 71.46545854521419,
       "cc_by_type": {
        "choice": 73.45132743362832,
        "noul": 71.42857142857142,
        "score": 69.51647677344282
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 71.46545854521419
      },
      "lead_generation": {
       "accuracy": 0.9259259259259259,
       "cc": 78.65664565502806,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 35.96993696508418
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 78.65664565502806
      },
      "legal_compliance": {
       "accuracy": 0.8666666666666667,
       "cc": 76.81444528324634,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 75,
        "score": 81.75912532342325
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 76.81444528324634
      },
      "llm_guardrails": {
       "accuracy": 0.7916666666666666,
       "cc": 72.51849215446394,
       "cc_by_type": {
        "choice": 83.60655737704917,
        "noul": 71.42857142857142,
        "score": 62.52034765777125
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 72.51849215446394
      },
      "model_routing": {
       "accuracy": 0.9261083743842364,
       "cc": 83.17577360184441,
       "cc_by_type": {
        "choice": 95.23273521816857,
        "noul": 100,
        "score": 54.294585587364665
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 83.17577360184441
      },
      "moderation_trust_safety": {
       "accuracy": 0.9166666666666666,
       "cc": 81.14209861079502,
       "cc_by_type": {
        "choice": 90.25974025974027,
        "noul": 66.66666666666667,
        "score": 86.49988890597811
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 81.14209861079502
      },
      "recruiting": {
       "accuracy": 0.7647058823529411,
       "cc": 66.11227351729069,
       "cc_by_type": {
        "choice": 62.382445141065844,
        "noul": 50,
        "score": 85.9543754108062
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 66.11227351729069
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 63.987715661714255,
       "cc_by_type": {
        "choice": 55.99022004889975,
        "noul": 100,
        "score": 35.97292693624302
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 63.987715661714255
      },
      "scientific_discovery": {
       "accuracy": 0.7619047619047619,
       "cc": 59.91676158127726,
       "cc_by_type": {
        "choice": 70.873786407767,
        "noul": 50,
        "score": 58.87649833606476
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 59.91676158127726
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 82.16687648931749,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 46.50062946795245
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 82.16687648931749
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.33110758644384,
      92.2500086656397
     ],
     "I": [
      67.09450470102875,
      75.25396599769745
     ],
     "capability": [
      77.88525338666481,
      83.2725659810476
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.10757097649757,
     "capability": 81.50321747405513,
     "intelligence": 71.89886397161271
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8546666666666667,
      "calibration": 89.6259235128982,
      "cc": 81.18490956865303,
      "cc_by_type": {
       "choice": 81.18490956865303
      },
      "cc_tiered": 79.1190223529297,
      "cc_tiered_open": 76.92440600606106,
      "cc_tiered_sealed": 81.31363869979835,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.18490956865303
     },
     "noul": {
      "accuracy": 0.9013333333333333,
      "calibration": 92.55662788909783,
      "cc": 80.26666666666667,
      "cc_by_type": {
       "noul": 80.26666666666667
      },
      "cc_tiered": 81.26327352492908,
      "cc_tiered_open": 85.29951690821257,
      "cc_tiered_sealed": 77.2270301416456,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 80.26666666666667
     },
     "score": {
      "accuracy": 0.7226666666666667,
      "calibration": 91.1401615274967,
      "cc": 55.788639506853855,
      "cc_by_type": {
       "score": 55.788639506853855
      },
      "cc_tiered": 55.31429603697938,
      "cc_tiered_open": 59.61386177168524,
      "cc_tiered_sealed": 51.01473030227352,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 55.788639506853855
     }
    },
    "raw_on_A": {
     "calibration": 90.61114153337692,
     "capability": 81.96187686080913,
     "intelligence": 73.31261218824132
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.020000000000000018,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.8836073417276,
    "Emphasis on Accuracy 60:20:20": 70.058843673427,
    "Emphasis on Cost 20:20:60": 62.3184485672163,
    "Emphasis on Speed 20:60:20": 75.577803207739,
    "Intelligence only": 71.89886397161271
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "jgeuter (Hugging Face)",
   "axes": {
    "calibration": 91.95478386454448,
    "cost": 58.304016843105764,
    "intelligence": 84.4898380044484,
    "speed": 72.19753970140297
   },
   "base_model": "google/gemma-4-31B-it",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07272856513224103,
      "ece_score": 85.4542869735518,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 354,
      "n_tvd": 0,
      "score": 85.4542869735518,
      "score_before_time_limit": 85.4542869735518,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.028984574860606686,
      "ece": 0.027592483833569454,
      "n": 375,
      "score": 94.48150323328612,
      "score_before_time_limit": 94.48150323328612,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.968,
      "interval90_width": 0.08333333333333333,
      "n": 375,
      "nrps": 0.022636592638970957,
      "score": 95.92856138679551,
      "score_before_time_limit": 95.92856138679551,
      "time_limit_failed_share": 0,
      "top_ece": 0.02939608981255941
     }
    },
    "score": 91.95478386454448
   },
   "capability": 88.22231093449645,
   "code_revision": "The author's own serving path ran unmodified: j-geuter/Jeff-JevBench@3b53b463f6427f45450b1e99c9d67842fed780d6 installing jeff 1.4.0 (j-geuter/Jeff@47ac025a1d8a011bdb1fb1cbbbf20607884d35ee). Native /v1/systemone endpoint with no response shim or instruction conversion. Reasoning enabled, JevBench limits enabled, fast_reasoning resolved to shared, reasoning_engine transformers, batching off, shipped Jeff-1.1-Large settings unchanged (settings sha256 75b684ae). Reasoning budget 128, shares choice 0 / noul 0.2 / score 0.6, maximum output per decision 35, maximum share 0.45, decision_seconds 8.5, request_seconds 30, window 1000 and warm-up 100 decisions. The commitment band moves probabilities strictly inside (0.2, 0.8) to 0.801 or 0.199. Adapted google/gemma-4-31B-it with adapter and decision head (jgeuter/Jeff-1.1-Large@970c21af, 29 files hash-verified), maximum input 8,192 tokens, no quantisation. Hash-locked dependencies included torch 2.13.0+cu130 and transformers 5.17.0; no vLLM or flash-linear-attention present. Source and weights ran on isolated rented hardware with networking disabled, no credentials and read-only mounts. The scored process was fresh; no earlier probes affected its process-wide running limits.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 589,108 input + 21,722 output tokens over 1,489 decisions = 410.228341169 tokens/decision. GPU-time cost USD 0.0245395223550721/1,000; old list-price cost USD 0.0405676292813969/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0245395223550721/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.042161033333333334
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02453952235507212
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "First measurement of jgeuter/Jeff-1.1-Large, a Jev-compatible model. Jeff-1.0-Large, Jeff-1.1-Medium and Jeff-1.1-Qwen-4B are separate measured models; nothing is inferred between them. DRAW: measured on the 27 Oct 2026 regular draw (fresh sealed items; not yet equated to the reference scale, so its numbers are shown as published, offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own draw. The author's own serving path ran unmodified with native choice, noul and score probabilities. The shipped configuration uses a one-pass decision head and limited generated reasoning read back into the answer: 186 decisions were reasoned, none dropped, and 1,314 were one-pass (21,830 reasoning tokens). Fast reasoning resolved to shared on this hardware (no room for a second copy of the weights); a card with room for a second copy could use the copy path, so this result is for this hardware. The settings target the published latency and cost caps: decision_seconds 8.5, max_share 0.45 and max_output_per_decision 35. The commitment band changes yes/no probabilities; Calibration includes its effect. No alternative configuration was measured. The card reports public-231 scores and separately held-out tuning data; the data used to tune the JevBench-specific settings are not identified. Training data are not identified beyond the card's description of CC BY-NC and research-only sources; the author states no JevBench items were used. Training examples and overlap with JevBench were not independently checked. Question-aware retrieval answered the 5 long items at at most 8,192 input tokens, selecting state passages matching the question in original order, with a start-of-state fallback. These answers use retrieved passages, not the whole state. All 1,500 requests returned HTTP 200 with answers, no refusals or errors; zero exceeded 10 s (maximum 8.319 s, under the author's own 8.5 s decision limit). Measured p50 0.106 s and p95 8.269 s; the reasoned decisions are the slow tail; the common self-hosted latency assumption is twice p50 plus 0.15 s. Native usage is 620,215 input and 21,830 output tokens. Ordinary input counts match the independent count on all 1,495 ordinary items; the 5 retrieval answers stay within 8,192 tokens. The server counts one-pass input only: an uncounted second-prefill estimate totals 199,718 tokens (88,572 generation-prefill plus 111,146 read-prefill), not measured natively and not priced. Native nonzero output is generated reasoning and is priced as reported. DETERMINISM (record-only): in the pre-sealed smoke, one repeated probe, a short score question that took the reasoning path, returned different probability vectors on its two repeats; no other probe differed, and the forced-reasoning process showed none. Weights carry non-commercial CC BY-NC-4.0 terms, the same terms as the Jeff-1.0-Large row; listed for evaluation only. HARDWARE: one Verda RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, EU); latency reflects that card.",
   "display": "Jeff-1.1-Large (jgeuter)",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-27",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.14",
    "results_sha256": "9e460d48571eeb00d94f554a985fdc2ebd52b63beb9aed7a4efe88762eb7b32e",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Measured 27 Oct 2026 on one rented Verda NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, Finland, EU) over loopback. One measured attempt in a fresh process with pinned weights and serving code, read-only model and source, no credentials and networking disabled. The standard self-hosted x2 + 0.15 s latency adjustment applies. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 79.90885261158905,
    "I_sealed": 89.07082339730775,
    "base": 84.4898380044484,
    "excess": -11.735358428156937,
    "gap": -9.161970785718694,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 80.05546732720302,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 79.23875432525952,
       "judge": 76.26895187870798,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 94.77777777777777,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 100,
       "judge": 89.99999999999999,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 64.89331272978637,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 79.3982419590397,
       "hard": 74.55892960469161,
       "judge": 58.951799005676584,
       "standard": 47.221884951513914
      }
     },
     "sealed|choice": {
      "cc": 88.96231891409286,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 98.03471994759252,
       "hard": 81.45996196915276,
       "judge": 94.82312338222607,
       "standard": 90.63962558502342
      }
     },
     "sealed|noul": {
      "cc": 89.70167100227538,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 97.70114942528735,
       "hard": 80.48780487804879,
       "judge": 95.55555555555554,
       "standard": 95.34883720930232
      }
     },
     "sealed|score": {
      "cc": 88.54848027555505,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 94.53891511372072,
       "hard": 79.81971665734804,
       "judge": 93.46156499152164,
       "standard": 95.64116301893631
      }
     }
    }
   },
   "jevbench_score": 74.46984358527004,
   "key": "jeff_1_1_large",
   "last_measured_on": "2026-10-27",
   "licence": "Weights CC BY-NC-4.0 per the model card (non-commercial; listed for evaluation only); base Apache-2.0; serving code Apache-2.0 (jeff 1.4.0).",
   "listing": "unranked",
   "measured_in": "v1.7.14",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "jgeuter/Jeff-1.1-Large@970c21af246a44c5527f7109fca129ab6015c941",
   "new_in": "v1.7.14",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale: its first anchor runs used models weaker than this row, and stronger anchors are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.9626666666666667,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/jgeuter/Jeff-1.1-Large",
   "scores": {
    "A": 74.46984358527004,
    "B": 76.27909333062428,
    "C": 74.46984358527004
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.361273870000241,
    "p50_s_raw": 0.1056369350001205,
    "p95_s_adjusted": 16.68819933229997,
    "p95_s_raw": 8.269099666149986
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 100
      },
      "calibration_gen": {
       "accuracy": 0.6428571428571429,
       "cc": 66.0402224291641,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 100,
        "score": 50.06871923554428
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 66.0402224291641
      },
      "composite_gen": {
       "accuracy": 0.9473684210526315,
       "cc": 93.2522101952364,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 79.75663058570919
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 93.2522101952364
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.85479817687114,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.56439453061344
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 99.85479817687114
      },
      "jaggedness": {
       "accuracy": 0.7272727272727273,
       "cc": 60.373522525097144,
       "cc_by_type": {
        "choice": 87.65432098765433,
        "noul": 33.33333333333333,
        "score": 60.13291325430378
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 60.373522525097144
      },
      "llm_router": {
       "accuracy": 0.9,
       "cc": 85.15260990808348,
       "cc_by_type": {
        "choice": 92.06815511163339,
        "noul": 100,
        "score": 63.38967461261702
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 85.15260990808348
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 100
      },
      "multilingual": {
       "accuracy": 0.9272529858849077,
       "cc": 89.07701563686811,
       "cc_by_type": {
        "choice": 86.74283250700583,
        "noul": 91.80327868852459,
        "score": 88.68493571507392
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 89.07701563686811
      },
      "negation_pair_gen": {
       "accuracy": 0.9090909090909091,
       "cc": 86.66666666666667,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 100
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 86.66666666666667
      },
      "option_order": {
       "accuracy": 0.8888888888888888,
       "cc": 85.71428571428571,
       "cc_by_type": {
        "choice": 85.71428571428571
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 85.71428571428571
      },
      "perturb_pair": {
       "accuracy": 0.9285714285714286,
       "cc": 92.5107573404163,
       "cc_by_type": {
        "choice": 81.13207547169812,
        "noul": 100,
        "score": 96.4001965495508
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 92.5107573404163
      },
      "usecase": {
       "accuracy": 0.916923076923077,
       "cc": 89.49981078234423,
       "cc_by_type": {
        "choice": 88.32968636032092,
        "noul": 93.84615384615384,
        "score": 86.32359214055786
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 89.49981078234423
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.9047619047619048,
       "cc": 88.935562678788,
       "cc_by_type": {
        "choice": 86.3013698630137,
        "noul": 100,
        "score": 80.50531817335033
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 88.935562678788
      },
      "cs": {
       "accuracy": 0.8648648648648649,
       "cc": 80.37239289279897,
       "cc_by_type": {
        "choice": 83.9572192513369,
        "noul": 77.77777777777777,
        "score": 79.38218164928224
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 80.37239289279897
      },
      "da": {
       "accuracy": 0.9166666666666666,
       "cc": 82.60021399533503,
       "cc_by_type": {
        "choice": 85.09316770186335,
        "noul": 100,
        "score": 62.70747428414174
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 82.60021399533503
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 85.81812582468916,
       "cc_by_type": {
        "choice": 95.36917931566761,
        "noul": 75,
        "score": 87.08519815839986
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 85.81812582468916
      },
      "el": {
       "accuracy": 0.9642857142857143,
       "cc": 94.71720444208518,
       "cc_by_type": {
        "choice": 88.46153846153844,
        "noul": 100,
        "score": 95.69007486471706
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 94.71720444208518
      },
      "en": {
       "accuracy": 0.9009009009009009,
       "cc": 87.58149408436036,
       "cc_by_type": {
        "choice": 87.10847975553857,
        "noul": 93.44262295081967,
        "score": 82.19337954672285
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 87.58149408436036
      },
      "es": {
       "accuracy": 0.918918918918919,
       "cc": 85.27695785514959,
       "cc_by_type": {
        "choice": 92.78267842822774,
        "noul": 77.77777777777777,
        "score": 85.27041735944327
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 85.27695785514959
      },
      "fi": {
       "accuracy": 0.9230769230769231,
       "cc": 91.44822668689825,
       "cc_by_type": {
        "choice": 86.28571428571429,
        "noul": 100,
        "score": 88.05896577498049
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 91.44822668689825
      },
      "fr": {
       "accuracy": 1,
       "cc": 97.79434039676369,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 93.38302119029106
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 97.79434039676369
      },
      "hi": {
       "accuracy": 0.8918918918918919,
       "cc": 88.88356847287271,
       "cc_by_type": {
        "choice": 75.20661157024793,
        "noul": 100,
        "score": 91.44409384837022
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 88.88356847287271
      },
      "id": {
       "accuracy": 0.975609756097561,
       "cc": 96.23460125976733,
       "cc_by_type": {
        "choice": 90.78341013824885,
        "noul": 100,
        "score": 97.92039364105312
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 96.23460125976733
      },
      "it": {
       "accuracy": 1,
       "cc": 98.09694393481725,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 94.29083180445174
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 98.09694393481725
      },
      "ja": {
       "accuracy": 0.9318181818181818,
       "cc": 89.8653417442394,
       "cc_by_type": {
        "choice": 84.45595854922279,
        "noul": 100,
        "score": 85.1400666834954
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 89.8653417442394
      },
      "ko": {
       "accuracy": 0.8837209302325582,
       "cc": 77.54621187112623,
       "cc_by_type": {
        "choice": 77.94117647058825,
        "noul": 60.00000000000001,
        "score": 94.69745914279048
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 77.54621187112623
      },
      "mixed": {
       "accuracy": 0.96875,
       "cc": 95.91682903881889,
       "cc_by_type": {
        "choice": 89.1989198919892,
        "noul": 100,
        "score": 98.55156722446746
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 95.91682903881889
      },
      "nl": {
       "accuracy": 0.9166666666666666,
       "cc": 77.8490637617958,
       "cc_by_type": {
        "choice": 92.92452830188678,
        "noul": 80,
        "score": 60.62266298350065
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 77.8490637617958
      },
      "no": {
       "accuracy": 0.9019607843137255,
       "cc": 86.78044084107269,
       "cc_by_type": {
        "choice": 79.34595524956971,
        "noul": 100,
        "score": 80.99536727364834
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 86.78044084107269
      },
      "pl": {
       "accuracy": 0.9122807017543859,
       "cc": 80.12540987186003,
       "cc_by_type": {
        "choice": 90.42043640234168,
        "noul": 88.23529411764706,
        "score": 61.72049909559134
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 80.12540987186003
      },
      "pt": {
       "accuracy": 0.9148936170212766,
       "cc": 82.39442476277065,
       "cc_by_type": {
        "choice": 88.01996672212977,
        "noul": 66.66666666666667,
        "score": 92.49664089951553
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 82.39442476277065
      },
      "sv": {
       "accuracy": 0.8913043478260869,
       "cc": 81.41905457359424,
       "cc_by_type": {
        "choice": 73.18435754189943,
        "noul": 100,
        "score": 71.07280617888328
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 81.41905457359424
      },
      "tr": {
       "accuracy": 0.9347826086956522,
       "cc": 91.9065296456147,
       "cc_by_type": {
        "choice": 79.47548460661345,
        "noul": 100,
        "score": 96.24410433023064
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 91.9065296456147
      },
      "uk": {
       "accuracy": 0.9285714285714286,
       "cc": 89.03439878447792,
       "cc_by_type": {
        "choice": 89.04109589041097,
        "noul": 100,
        "score": 78.06210046302279
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 89.03439878447792
      },
      "zh": {
       "accuracy": 0.8780487804878049,
       "cc": 82.52405170301772,
       "cc_by_type": {
        "choice": 81.40495867768595,
        "noul": 75,
        "score": 91.16719643136719
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 82.52405170301772
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.904,
       "cc": 86.90525507471884,
       "cc_by_type": {
        "choice": 86.90525507471884
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 86.90525507471884
      },
      "noul": {
       "accuracy": 0.9626666666666667,
       "cc": 92.53333333333333,
       "cc_by_type": {
        "noul": 92.53333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 92.53333333333333
      },
      "score": {
       "accuracy": 0.896,
       "cc": 85.66875525071255,
       "cc_by_type": {
        "score": 85.66875525071255
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 85.66875525071255
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.9166666666666666,
       "cc": 83.05920657790115,
       "cc_by_type": {
        "choice": 91.10452186805038,
        "noul": 71.42857142857142,
        "score": 86.64452643708162
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 83.05920657790115
      },
      "code_linting": {
       "accuracy": 0.9710144927536232,
       "cc": 97.40125211593954,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 92.20375634781861
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 97.40125211593954
      },
      "customer_support": {
       "accuracy": 0.9696969696969697,
       "cc": 93.9783449897734,
       "cc_by_type": {
        "choice": 96.309963099631,
        "noul": 86.66666666666667,
        "score": 98.95840520302251
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 93.9783449897734
      },
      "demand_forecasting": {
       "accuracy": 0.7424242424242424,
       "cc": 58.175259947882154,
       "cc_by_type": {
        "choice": 49.96209249431388,
        "noul": 53.846153846153854,
        "score": 70.71753350317873
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 58.175259947882154
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.967741935483871,
       "cc": 94.5875506131572,
       "cc_by_type": {
        "choice": 95.77762139338495,
        "noul": 100,
        "score": 87.9850304460867
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 94.5875506131572
      },
      "feature_extraction": {
       "accuracy": 0.9305555555555556,
       "cc": 88.39059272296247,
       "cc_by_type": {
        "choice": 84.92462311557789,
        "noul": 85.71428571428572,
        "score": 94.53286933902375
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 88.39059272296247
      },
      "financial_crime": {
       "accuracy": 0.9047619047619048,
       "cc": 82.2641984705386,
       "cc_by_type": {
        "choice": 87.24082934609251,
        "noul": 100,
        "score": 59.55176606552328
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 82.2641984705386
      },
      "gaming": {
       "accuracy": 0.8620689655172413,
       "cc": 87.175206912686,
       "cc_by_type": {
        "choice": 85.65737051792829,
        "noul": 100,
        "score": 75.86825022012968
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 87.175206912686
      },
      "insurance_claims": {
       "accuracy": 0.8548387096774194,
       "cc": 78.98078246165217,
       "cc_by_type": {
        "choice": 73.2739420935412,
        "noul": 76.47058823529412,
        "score": 87.19781705612118
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 78.98078246165217
      },
      "knowledge_graphs": {
       "accuracy": 0.9701492537313433,
       "cc": 90.93202685893942,
       "cc_by_type": {
        "choice": 95.50224887556223,
        "noul": 84.61538461538463,
        "score": 92.67844708587138
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 90.93202685893942
      },
      "lead_generation": {
       "accuracy": 0.9466666666666667,
       "cc": 92.25767620817597,
       "cc_by_type": {
        "choice": 87.30606488011283,
        "noul": 94.28571428571428,
        "score": 95.18124945870079
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 92.25767620817597
      },
      "legal_compliance": {
       "accuracy": 0.9107142857142857,
       "cc": 89.50956494274465,
       "cc_by_type": {
        "choice": 87.96389167502507,
        "noul": 100,
        "score": 80.56480315320889
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 89.50956494274465
      },
      "llm_guardrails": {
       "accuracy": 0.9859154929577465,
       "cc": 96.75246673876718,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 90.25740021630153
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 96.75246673876718
      },
      "model_routing": {
       "accuracy": 0.9256756756756757,
       "cc": 88.40654413002011,
       "cc_by_type": {
        "choice": 93.76839189890946,
        "noul": 100,
        "score": 71.45124049115087
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 88.40654413002011
      },
      "moderation_trust_safety": {
       "accuracy": 0.9661016949152542,
       "cc": 95.00480884330581,
       "cc_by_type": {
        "choice": 86.92810457516339,
        "noul": 100,
        "score": 98.08632195475407
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 95.00480884330581
      },
      "other": {
       "accuracy": 0.875,
       "cc": 84.73002896736108,
       "cc_by_type": {
        "choice": 80.02219755826859,
        "noul": 100,
        "score": 74.16788934381464
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 84.73002896736108
      },
      "recruiting": {
       "accuracy": 0.9230769230769231,
       "cc": 77.21226285030001,
       "cc_by_type": {
        "choice": 87.0223503965393,
        "noul": 100,
        "score": 44.614438154360705
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 77.21226285030001
      },
      "risk_assessment": {
       "accuracy": 0.875,
       "cc": 86.30075929263121,
       "cc_by_type": {
        "choice": 62.34309623430962,
        "noul": 100,
        "score": 96.55918164358404
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 86.30075929263121
      },
      "scientific_discovery": {
       "accuracy": 0.9285714285714286,
       "cc": 83.53321258813867,
       "cc_by_type": {
        "choice": 92.63803680981594,
        "noul": 60.00000000000001,
        "score": 97.96160095460006
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 83.53321258813867
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.73924320402602,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.21772961207809
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 99.73924320402602
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.78262449508242,
      93.02378566444065
     ],
     "I": [
      80.6440662485624,
      87.26561009692087
     ],
     "capability": [
      85.40992674922313,
      89.53648257568206
     ],
     "composite": null,
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.95478386454448,
     "capability": 88.22231093449645,
     "intelligence": 84.4898380044484
    },
    "per_type": {
     "choice": {
      "accuracy": 0.904,
      "calibration": 85.4542869735518,
      "cc": 86.90525507471884,
      "cc_by_type": {
       "choice": 86.90525507471884
      },
      "cc_tiered": 84.50889312064794,
      "cc_tiered_open": 80.05546732720302,
      "cc_tiered_sealed": 88.96231891409286,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 86.90525507471884
     },
     "noul": {
      "accuracy": 0.9626666666666667,
      "calibration": 94.48150323328612,
      "cc": 92.53333333333333,
      "cc_by_type": {
       "noul": 92.53333333333333
      },
      "cc_tiered": 92.23972439002657,
      "cc_tiered_open": 94.77777777777777,
      "cc_tiered_sealed": 89.70167100227538,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 92.53333333333333
     },
     "score": {
      "accuracy": 0.896,
      "calibration": 95.92856138679551,
      "cc": 85.66875525071255,
      "cc_by_type": {
       "score": 85.66875525071255
      },
      "cc_tiered": 76.72089650267071,
      "cc_tiered_open": 64.89331272978637,
      "cc_tiered_sealed": 88.54848027555505,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 85.66875525071255
     }
    },
    "raw_on_A": {
     "calibration": 91.23476486856018,
     "capability": 86.8257801906185,
     "intelligence": 82.41679551267683
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.03110518102163,
    "Emphasis on Accuracy 60:20:20": 75.17712506416483,
    "Emphasis on Cost 20:20:60": 64.81632379199262,
    "Emphasis on Speed 20:60:20": 70.88188745693627,
    "Intelligence only": 84.4898380044484
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "jgeuter (Hugging Face)",
   "axes": {
    "calibration": 89.2826256096074,
    "cost": 66.15113749144632,
    "intelligence": 74.49393760932514,
    "speed": 73.9814920153467
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03660587736612005,
      "ece_score": 92.678824526776,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 92.678824526776,
      "score_before_time_limit": 92.678824526776,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.08204756078344574,
      "ece": 0.039414676672970925,
      "n": 375,
      "score": 92.1170646654058,
      "score_before_time_limit": 92.1170646654058,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8693333333333333,
      "interval90_width": 0.10555555555555553,
      "n": 375,
      "nrps": 0.06185082837121522,
      "score": 83.05198763664039,
      "score_before_time_limit": 83.05198763664039,
      "time_limit_failed_share": 0,
      "top_ece": 0.13855470944798848
     }
    },
    "score": 89.2826256096074
   },
   "capability": 81.88828160946628,
   "code_revision": "The author's own serving path ran unmodified: j-geuter/Jeff-JevBench@3b53b463f6427f45450b1e99c9d67842fed780d6 installing jeff 1.4.0 (j-geuter/Jeff@47ac025a1d8a011bdb1fb1cbbbf20607884d35ee). Native /v1/systemone endpoint with no response shim or instruction conversion. Reasoning enabled, JevBench limits enabled, fast_reasoning resolved to copy, reasoning_engine transformers, batching off, shipped Jeff-1.1-Medium settings unchanged. Reasoning budget 256, shares choice 0 / noul 0.1 / score 0.6, maximum output per decision 224, maximum share 0.45, decision_seconds 8.5, request_seconds 30, window 1000 and warm-up 100 decisions. The commitment band moves probabilities strictly inside (0.2, 0.8) to 0.801 or 0.199. Adapted google/gemma-4-12B-it with adapter and decision head, full bidirectional attention in the decision pass, maximum input 8,192 tokens, no quantisation. Hash-locked dependencies included torch 2.13.0+cu130 and transformers 5.17.0. Source and weights ran on isolated rented hardware with networking disabled, no credentials and read-only mounts. The scored process was fresh; no earlier probes affected its process-wide running limits.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 814,647 input + 71,931 output tokens over 1,485 decisions = 597.022222222 tokens/decision. GPU-time cost USD 0.0134367865102517/1,000; old list-price cost USD 0.0346949494949495/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0134367865102517/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.03757996666666667
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013436786510251734
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "First measurement of jgeuter/Jeff-1.1-Medium, a Jev-compatible model. Jeff-1.0-Large remains a separate measured model; nothing is inferred about unmeasured siblings. The author's own serving path ran unmodified with native choice, noul and score probabilities. The shipped configuration uses a one-pass decision head and limited generated reasoning read back into the answer: 324 decisions were reasoned, three further reasonings were admitted but dropped, and 1,173 were one-pass. Fast reasoning resolved to copy on this hardware; other hardware may use a slower shared path. The settings target the published latency and cost caps: decision_seconds 8.5, max_share 0.45 and max_output_per_decision 224. The commitment band changes yes/no probabilities; Calibration includes its effect. No alternative configuration was measured. The card reports public-231 scores and separately held-out tuning data; the data used to tune the JevBench-specific settings are not identified. Every shipped temperature is a point of one geometric grid (steps of 1.05), and the reasoning thresholds are entries of the validation-quantile table published with the model for the chosen shares, so the many-digit values are grid and table entries, not item-level fits. Training data are not identified beyond the card's description of CC BY-NC and research-only sources; the author states no JevBench items were used. Training examples and overlap with JevBench were not independently checked. Question-aware retrieval answered the 15 long items at at most 8,192 input tokens, selecting state passages matching the question in original order, with a start-of-state fallback. These answers use retrieved passages, not the whole state. All 1,500 requests returned HTTP 200 with answers, no refusals or errors; zero exceeded 10 s (maximum 8.092 s). Measured p50 0.067 s and p95 6.957 s; the common self-hosted latency assumption is twice p50 plus 0.15 s. Native usage is 910,751 input and 72,216 output tokens. Ordinary input counts match the independent count on all 1,485 common items; the 15 retrieval answers stay within 8,192 tokens. The server counts one-pass input only: an uncounted second-prefill estimate totals 427,166 tokens (188,995 generation-prefill plus 238,171 read-prefill), not measured natively and not priced. Native nonzero output is generated reasoning and is priced as reported. Cost is an ESTIMATE using explicit 0.05/0.15 USD per million tokens, not charged usage or a host quote. The common 1,485-item basis costs USD 0.03469 per 1,000 decisions (0.537 times cap); all 1,500 cost USD 0.03758 (0.582 times cap). Weights carry non-commercial CC BY-NC-4.0 terms; listed for evaluation only. If the uncounted prefills were priced as input (403,115 tokens on the common 1,485-item basis), the common-basis cost would be USD 0.048268 per 1,000 decisions, 0.747 times the cap (all-1,500 USD 0.051819, 0.802 times the cap). The Cost axis would fall from 53.79 to 49.49, below the 50-point composite gate. This sensitivity estimate is not included in the published price or scores. Scoring the 15 retrieval answers as misses instead would lower Capability by 0.41 points, from 81.89 to 81.48.",
   "display": "Jeff-1.1-Medium (jgeuter)",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-26",
    "release": "v1.7.8",
    "results_sha256": "ae6176c6e26c68f2e3634e2c59c019ff8be04dfdcfb3f2ac73905338ad25f747"
   },
   "endpoint_condition": "Measured 26 Oct 2026 on one rented NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB in Finland over loopback. One measured attempt in a fresh process with pinned weights and serving code, read-only model and source, no credentials and networking disabled. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 75.96270449682046,
    "I_sealed": 73.02517072182982,
    "base": 74.49393760932514,
    "excess": -6.90199299536507,
    "gap": 2.9375337749906407,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.89450831799229,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 73.57659641396664,
       "judge": 76.26895187870798,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 82.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 80,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 69.394571356044,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 82.09429427667607,
       "hard": 67.98331028970999,
       "judge": 66.81081924529522,
       "standard": 69.74286019451912
      }
     },
     "sealed|choice": {
      "cc": 74.82919916715036,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 71.89776801202729,
       "judge": 70.02938295788442,
       "standard": 79.11832946635731
      }
     },
     "sealed|noul": {
      "cc": 76.91360415189546,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 67.27272727272727,
       "judge": 81.53846153846153,
       "standard": 88.05970149253733
      }
     },
     "sealed|score": {
      "cc": 67.33270884644364,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 80.6214036297183,
       "hard": 65.80169539771485,
       "judge": 64.5447063320077,
       "standard": 67.93239212391775
      }
     }
    }
   },
   "jevbench_score": 75.10595924834655,
   "key": "jeff_1_1_medium",
   "last_measured_on": "2026-10-26",
   "licence": "Weights CC BY-NC-4.0 per the model card (non-commercial; listed for evaluation only); bases Apache-2.0; serving code Apache-2.0 (jeff 1.4.0).",
   "listing": "ranked",
   "measured_in": "v1.7.8",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "jgeuter/Jeff-1.1-Medium@540c40924dd67c715d3367309ba6744bac754b00",
   "new_in": "v1.7.8",
   "noul_decisive": {
    "acc_among_decisive": 0.8933333333333333,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 18,
   "ranked": true,
   "ranks": {
    "A": 18,
    "B": 13,
    "C": 17,
    "capability": 21
   },
   "repo": "https://huggingface.co/jgeuter/Jeff-1.1-Medium",
   "scores": {
    "A": 75.10595924834655,
    "B": 74.98275172815433,
    "C": 75.10595924834655
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2842651589998241,
    "p50_s_raw": 0.06713257949991203,
    "p95_s_adjusted": 14.064592863100074,
    "p95_s_raw": 6.957296431550037
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 74.04529521973119,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 38.80255232586025
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 74.04529521973119
      },
      "calibration_gen": {
       "accuracy": 0.7821782178217822,
       "cc": 73.26356147477776,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 84.00000000000001,
        "score": 83.51072684214877
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 73.26356147477776
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 86.32227320831622,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 77.77777777777777,
        "score": 97.44920444879689
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 86.32227320831622
      },
      "fairness": {
       "accuracy": 1,
       "cc": 98.94166903836147,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.82500711508443
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 98.94166903836147
      },
      "jaggedness": {
       "accuracy": 0.7166666666666667,
       "cc": 60.46415936626732,
       "cc_by_type": {
        "choice": 61.02021174205967,
        "noul": 78.37837837837837,
        "score": 41.9938879783639
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 60.46415936626732
      },
      "llm_router": {
       "accuracy": 0.8594249201277955,
       "cc": 78.38340387498164,
       "cc_by_type": {
        "choice": 81.77335563969358,
        "noul": 91.75257731958763,
        "score": 61.62427866566371
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 78.38340387498164
      },
      "long_state": {
       "accuracy": 0.7333333333333333,
       "cc": 28.483908704969963,
       "cc_by_type": {
        "choice": 74.19354838709678,
        "noul": 0,
        "score": 11.258177727813102
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 28.483908704969963
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 65.70296712637615,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 50,
        "score": 84.22230344098408
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 65.70296712637615
      },
      "multilingual": {
       "accuracy": 0.8036363636363636,
       "cc": 69.89465982016291,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": 69.44444444444444,
        "score": 68.60601692018513
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.89465982016291
      },
      "negation_pair_gen": {
       "accuracy": 0.8604651162790697,
       "cc": 74.24798945628622,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 39.99999999999999,
        "score": 89.41063503552536
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.24798945628622
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 85.66876756627528,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100,
        "score": 78.02732371984685
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.66876756627528
      },
      "usecase": {
       "accuracy": 0.7727272727272727,
       "cc": 72.53328986838547,
       "cc_by_type": {
        "choice": 71.81372549019609,
        "noul": 83.33333333333333,
        "score": 62.45281078162699
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 72.53328986838547
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 52.33345053481445,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": 64.14320874730049
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 52.33345053481445
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 78.31837894245962,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 55.69246401631893
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 78.31837894245962
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 93.77361445100517,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 97.25004689283854
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 93.77361445100517
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.58948950019511,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.62135490654435
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.58948950019511
      },
      "en": {
       "accuracy": 0.8288444830582102,
       "cc": 75.40805527369746,
       "cc_by_type": {
        "choice": 77.15429394320303,
        "noul": 80.57553956834533,
        "score": 68.49433230954402
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 75.40805527369746
      },
      "es": {
       "accuracy": 0.9032258064516129,
       "cc": 81.97978510361118,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 55.72728353269325
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 81.97978510361118
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 55.50725895726558,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 98.76221403026668
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 55.50725895726558
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 72.88174853616643,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 82.17465737320516
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 72.88174853616643
      },
      "hi": {
       "accuracy": 0.9523809523809523,
       "cc": 88.07505097668599,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 75,
        "score": 89.22515293005797
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 88.07505097668599
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 84.94064835071737,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 68.85156313477098
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 84.94064835071737
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 28.694620754724344,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 39.99999999999999,
        "score": 17.569168062425632
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 28.694620754724344
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 51.38820512271098,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 54.16461536813292
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 51.38820512271098
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.96951944444963,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.90855833334889
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.96951944444963
      },
      "nl": {
       "accuracy": 0.8,
       "cc": 51.54853464698423,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 21.312270607619375
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 51.54853464698423
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8974358974358975,
       "cc": 88.22581168041158,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 100,
        "score": 72.29487050462875
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 88.22581168041158
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 76.18822971124843,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 70.99250292278478
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 76.18822971124843
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 22.60823352443819,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": 60.00000000000001,
        "score": 9.952360147782645
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 22.60823352443819
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 47.06918621687356,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": -8.792441349379331
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 47.06918621687356
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 70.84238776953744,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 77.04329234087038
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 70.84238776953744
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 76.5651301216297,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 96.36205703155578
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 76.5651301216297
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8146666666666667,
       "cc": 76.06153963910522,
       "cc_by_type": {
        "choice": 76.06153963910522
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 76.06153963910522
      },
      "noul": {
       "accuracy": 0.8933333333333333,
       "cc": 78.66666666666666,
       "cc_by_type": {
        "noul": 78.66666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 78.66666666666666
      },
      "score": {
       "accuracy": 0.7733333333333333,
       "cc": 68.37938316437497,
       "cc_by_type": {
        "score": 68.37938316437497
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 68.37938316437497
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6818181818181818,
       "cc": 50.997318254623764,
       "cc_by_type": {
        "choice": 68,
        "noul": 25,
        "score": 59.99195476387129
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 50.997318254623764
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 75.59952536520309,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 45.43211646828009
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 75.59952536520309
      },
      "customer_support": {
       "accuracy": 0.9285714285714286,
       "cc": 93.82613783345033,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100,
        "score": 97.738576101977
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 93.82613783345033
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 37.02863066579305,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 100,
        "score": -30.434575838878143
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 37.02863066579305
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 62.4594050673673,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 97.5231427383338
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 62.4594050673673
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": 41.24982350682995,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 100,
        "score": -25.93606407070512
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 41.24982350682995
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 51.987489008302326,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 42.51708887364644
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 51.987489008302326
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 52.81035605231892,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0,
        "score": 89.78995317437838
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 52.81035605231892
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 53.29289143844748,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 60.00000000000001,
        "score": 83.21200764867575
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 53.29289143844748
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 65.41736151061887,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100,
        "score": 34.7959817481307
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 65.41736151061887
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 70.6861526028783,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 100,
        "score": 35.31427176212328
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 70.6861526028783
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 18.13702390604988,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 17.851931933203403
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 18.13702390604988
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 81.29121408419138,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 100,
        "score": 55.38995703184477
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 81.29121408419138
      },
      "model_routing": {
       "accuracy": 0.8755760368663594,
       "cc": 80.18749530098063,
       "cc_by_type": {
        "choice": 88.31168831168831,
        "noul": 100,
        "score": 52.25079759125356
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.18749530098063
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 56.33976600319954,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 92.05192756813989
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 56.33976600319954
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 67.65913874047219,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 69.64408288808322
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 67.65913874047219
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 61.179241846697074,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50,
        "score": 63.094375786396625
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 61.179241846697074
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 52.34973530808358,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 81.68688708367102
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 52.34973530808358
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 92.54238789893508,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100,
        "score": 98.97947686406498
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 92.54238789893508
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.3699288964331,
      90.33992765291686
     ],
     "I": [
      70.30556786548495,
      78.00313122788796
     ],
     "capability": [
      78.07130351149658,
      83.70328873895242
     ],
     "composite": null,
     "seed": 36
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.2826256096074,
     "capability": 81.88828160946628,
     "intelligence": 74.49393760932514
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8146666666666667,
      "calibration": 92.678824526776,
      "cc": 76.06153963910522,
      "cc_by_type": {
       "choice": 76.06153963910522
      },
      "cc_tiered": 75.36185374257133,
      "cc_tiered_open": 75.89450831799229,
      "cc_tiered_sealed": 74.82919916715036,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.06153963910522
     },
     "noul": {
      "accuracy": 0.8933333333333333,
      "calibration": 92.1170646654058,
      "cc": 78.66666666666666,
      "cc_by_type": {
       "noul": 78.66666666666666
      },
      "cc_tiered": 79.75631898416029,
      "cc_tiered_open": 82.59903381642512,
      "cc_tiered_sealed": 76.91360415189546,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 78.66666666666666
     },
     "score": {
      "accuracy": 0.7733333333333333,
      "calibration": 83.05198763664039,
      "cc": 68.37938316437497,
      "cc_by_type": {
       "score": 68.37938316437497
      },
      "cc_tiered": 68.36364010124382,
      "cc_tiered_open": 69.394571356044,
      "cc_tiered_sealed": 67.33270884644364,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.37938316437497
     }
    },
    "raw_on_A": {
     "calibration": 89.32459776218087,
     "capability": 82.6725937543221,
     "intelligence": 76.0205897464633
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 71.33057107126643,
    "Emphasis on Accuracy 60:20:20": 72.56312102807571,
    "Emphasis on Cost 20:20:60": 69.16442512367286,
    "Emphasis on Speed 20:60:20": 72.36781073721271,
    "Intelligence only": 74.49393760932514
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "jgeuter (Hugging Face)",
   "axes": {
    "calibration": 91.10060876207879,
    "cost": 80.68251138779327,
    "intelligence": 68.16962043390559,
    "speed": 78.37104008757272
   },
   "base_model": "Qwen/Qwen3.5-4B",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05407828625149268,
      "ece_score": 89.18434274970146,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 354,
      "n_tvd": 0,
      "score": 89.18434274970146,
      "score_before_time_limit": 89.18434274970146,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.07577221725911298,
      "ece": 0.03297107777554944,
      "n": 375,
      "score": 93.40578444489012,
      "score_before_time_limit": 93.40578444489012,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.92,
      "interval90_width": 0.15177777777777782,
      "n": 375,
      "nrps": 0.050909861974440654,
      "score": 90.71169909164482,
      "score_before_time_limit": 90.71169909164482,
      "time_limit_failed_share": 0,
      "top_ece": 0.06742807809633145
     }
    },
    "score": 91.10060876207879
   },
   "capability": 79.63511459799219,
   "code_revision": "The author's own serving path ran unmodified: j-geuter/Jeff-JevBench@3b53b463f6427f45450b1e99c9d67842fed780d6 installing jeff 1.4.0 (j-geuter/Jeff@47ac025a1d8a011bdb1fb1cbbbf20607884d35ee). Native /v1/systemone endpoint with no response shim or instruction conversion. Reasoning enabled, JevBench limits enabled, fast_reasoning resolved to copy, reasoning_engine transformers, batching off, shipped Jeff-1.1-Qwen-4B settings unchanged (settings sha256 4b06e2ea). Reasoning budget 256, shares choice 0 / noul 0.1 / score 0.6, maximum output per decision 143, maximum share 0.45, decision_seconds 8.5, request_seconds 30, window 1000 and warm-up 100 decisions. The commitment band moves probabilities strictly inside (0.2, 0.8) to 0.801 or 0.199. Adapted Qwen/Qwen3.5-4B with adapter and decision head (jgeuter/Jeff-1.1-Qwen-4B@5401940b, 18 files hash-verified), full bidirectional attention in the decision pass, maximum input 8,192 tokens, no quantisation. Hash-locked dependencies included torch 2.13.0+cu130 and transformers 5.17.0; no vLLM or flash-linear-attention present, so the Qwen3.5 linear-attention layers ran without the optional kernels. Source and weights ran on isolated rented hardware with networking disabled, no credentials and read-only mounts. The scored process was fresh; no earlier probes affected its process-wide running limits.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 598,499 input + 68,954 output tokens over 1,489 decisions = 448.255876427 tokens/decision. GPU-time cost USD 0.00440469952037007/1,000; old list-price cost USD 0.0190047481531229/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00440469952037007/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.019557019999999998
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004404699520370065
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "First measurement of jgeuter/Jeff-1.1-Qwen-4B, a Jev-compatible model. Jeff-1.0-Large, Jeff-1.1-Medium and Jeff-1.1-Large are separate measured models; nothing is inferred between them. DRAW: measured on the 27 Oct 2026 regular draw (fresh sealed items; not yet equated to the reference scale, so its numbers are shown as published, offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own draw. The author's own serving path ran unmodified with native choice, noul and score probabilities. The shipped configuration uses a one-pass decision head and limited generated reasoning read back into the answer: 313 decisions were reasoned, none dropped, and 1,187 were one-pass (69,580 reasoning tokens). Fast reasoning resolved to copy on this hardware; other hardware may use a slower shared path. The settings target the published latency and cost caps: decision_seconds 8.5, max_share 0.45 and max_output_per_decision 143. The commitment band changes yes/no probabilities; Calibration includes its effect. No alternative configuration was measured. The card reports public-231 scores and separately held-out tuning data; the data used to tune the JevBench-specific settings are not identified. Training data are not identified beyond the card's description of CC BY-NC and research-only sources; the author states no JevBench items were used. Training examples and overlap with JevBench were not independently checked. Question-aware retrieval answered the 5 long items at at most 8,192 input tokens, selecting state passages matching the question in original order, with a start-of-state fallback. These answers use retrieved passages, not the whole state. All 1,500 requests returned HTTP 200 with answers, no refusals or errors; zero exceeded 10 s (maximum 5.921 s). Measured p50 0.039 s and p95 3.110 s; the common self-hosted latency assumption is twice p50 plus 0.15 s. Native usage is 629,951 input and 69,580 output tokens. Ordinary input counts match the independent count on all 1,495 ordinary items; the 5 retrieval answers stay within 8,192 tokens. The server counts one-pass input only: an uncounted second-prefill estimate totals 348,654 tokens (138,911 generation-prefill plus 209,743 read-prefill), not measured natively and not priced. Native nonzero output is generated reasoning and is priced as reported. Pre-sealed smoke repeats showed no non-deterministic probe. Weights carry non-commercial CC BY-NC-4.0 terms; listed for evaluation only. HARDWARE: one Verda RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, EU); latency reflects that card.",
   "display": "Jeff-1.1-Qwen-4B (jgeuter)",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-27",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.14",
    "results_sha256": "9e460d48571eeb00d94f554a985fdc2ebd52b63beb9aed7a4efe88762eb7b32e",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Measured 27 Oct 2026 on one rented Verda NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, Finland, EU) over loopback. One measured attempt in a fresh process with pinned weights and serving code, read-only model and source, no credentials and networking disabled. The standard self-hosted x2 + 0.15 s latency adjustment applies. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 63.56890005749781,
    "I_sealed": 72.77034081031337,
    "base": 68.16962043390559,
    "excess": -11.774828395253808,
    "gap": -9.201440752815564,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 69.12190578377826,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 60.36489462094999,
       "judge": 80.22412656558998,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 73.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 47.98576057229005,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.2914265120391,
       "hard": 65.54734003800944,
       "judge": 58.69175321689277,
       "standard": -13.349220295927356
      }
     },
     "sealed|choice": {
      "cc": 68.92482716639536,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 84.27775958074025,
       "hard": 57.690682442425526,
       "judge": 74.11561691113029,
       "standard": 75.93046579006018
      }
     },
     "sealed|noul": {
      "cc": 79.45406536663602,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 90.80459770114942,
       "hard": 63.414634146341456,
       "judge": 91.11111111111111,
       "standard": 88.37209302325581
      }
     },
     "sealed|score": {
      "cc": 69.93212989790872,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 92.15537922091248,
       "hard": 52.02043718072626,
       "judge": 81.22176375056004,
       "standard": 77.70943989179477
      }
     }
    }
   },
   "jevbench_score": 78.73978374847326,
   "key": "jeff_1_1_qwen_4b",
   "last_measured_on": "2026-10-27",
   "licence": "Weights CC BY-NC-4.0 per the model card (non-commercial; listed for evaluation only); base Apache-2.0; serving code Apache-2.0 (jeff 1.4.0).",
   "listing": "unranked",
   "measured_in": "v1.7.14",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "jgeuter/Jeff-1.1-Qwen-4B@5401940b5a8c959cba82187ebfa2dce5a21abc0a",
   "new_in": "v1.7.14",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale: its first anchor runs used models weaker than this row, and stronger anchors are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.9066666666666666,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/jgeuter/Jeff-1.1-Qwen-4B",
   "scores": {
    "A": 78.73978374847326,
    "B": 76.37140266154091,
    "C": 78.73978374847326
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22840540000006512,
    "p50_s_raw": 0.039202700000032564,
    "p95_s_adjusted": 6.3707362253999245,
    "p95_s_raw": 3.110368112699962
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.75,
       "cc": 66.66666666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 66.66666666666666
      },
      "calibration_gen": {
       "accuracy": 0.5714285714285714,
       "cc": 49.28919154818743,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": 100,
        "score": 25.78965256664022
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 49.28919154818743
      },
      "composite_gen": {
       "accuracy": 0.6842105263157895,
       "cc": 45.34136314731962,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": 19.999999999999996,
        "score": 69.35742277529219
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 45.34136314731962
      },
      "fairness": {
       "accuracy": 1,
       "cc": 98.65817048219975,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.97451144659925
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 98.65817048219975
      },
      "jaggedness": {
       "accuracy": 0.5909090909090909,
       "cc": 46.52304033058433,
       "cc_by_type": {
        "choice": 38.2716049382716,
        "noul": 33.33333333333333,
        "score": 67.96418272014807
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 46.52304033058433
      },
      "llm_router": {
       "accuracy": 0.8222222222222222,
       "cc": 69.7795942671967,
       "cc_by_type": {
        "choice": 78.84841363102231,
        "noul": 91.30434782608697,
        "score": 39.186021344480814
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 69.7795942671967
      },
      "long_state": {
       "accuracy": 0.6,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 67.74193548387098
      },
      "ml_usecase": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 100
      },
      "multilingual": {
       "accuracy": 0.8230184581976113,
       "cc": 75.26064963334186,
       "cc_by_type": {
        "choice": 67.01875404181938,
        "noul": 84.42622950819671,
        "score": 74.33696535000948
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 75.26064963334186
      },
      "negation_pair_gen": {
       "accuracy": 0.9545454545454546,
       "cc": 91.66666666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 83.33333333333333
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 91.66666666666666
      },
      "option_order": {
       "accuracy": 0.7777777777777778,
       "cc": 71.42857142857144,
       "cc_by_type": {
        "choice": 71.42857142857144
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 71.42857142857144
      },
      "perturb_pair": {
       "accuracy": 0.9285714285714286,
       "cc": 89.7320182783974,
       "cc_by_type": {
        "choice": 90.56603773584905,
        "noul": 80,
        "score": 98.63001709934316
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 89.7320182783974
      },
      "usecase": {
       "accuracy": 0.796923076923077,
       "cc": 69.48293775517924,
       "cc_by_type": {
        "choice": 66.44784828592267,
        "noul": 72.30769230769232,
        "score": 69.69327267192273
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 69.48293775517924
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.9523809523809523,
       "cc": 94.86365481720178,
       "cc_by_type": {
        "choice": 86.3013698630137,
        "noul": 100,
        "score": 98.28959458859163
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 94.86365481720178
      },
      "cs": {
       "accuracy": 0.8378378378378378,
       "cc": 74.26090484461821,
       "cc_by_type": {
        "choice": 67.91443850267379,
        "noul": 77.77777777777777,
        "score": 77.09049825340306
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 74.26090484461821
      },
      "da": {
       "accuracy": 0.7777777777777778,
       "cc": 66.09674563622328,
       "cc_by_type": {
        "choice": 55.27950310559007,
        "noul": 81.81818181818181,
        "score": 61.19255198489797
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 66.09674563622328
      },
      "de": {
       "accuracy": 0.875,
       "cc": 86.64785413142734,
       "cc_by_type": {
        "choice": 81.47671726267045,
        "noul": 100,
        "score": 78.46684513161158
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 86.64785413142734
      },
      "el": {
       "accuracy": 0.75,
       "cc": 62.171521388365186,
       "cc_by_type": {
        "choice": 53.84615384615385,
        "noul": 60.00000000000001,
        "score": 72.6684103189417
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 62.171521388365186
      },
      "en": {
       "accuracy": 0.8018018018018018,
       "cc": 70.4946659163481,
       "cc_by_type": {
        "choice": 69.91978609625669,
        "noul": 73.77049180327869,
        "score": 67.79371984950893
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 70.4946659163481
      },
      "es": {
       "accuracy": 0.8108108108108109,
       "cc": 66.9882524606622,
       "cc_by_type": {
        "choice": 71.13071371291097,
        "noul": 55.55555555555556,
        "score": 74.27848811352005
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 66.9882524606622
      },
      "fi": {
       "accuracy": 0.8269230769230769,
       "cc": 77.67276238071753,
       "cc_by_type": {
        "choice": 58.85714285714286,
        "noul": 89.4736842105263,
        "score": 84.68746007448343
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 77.67276238071753
      },
      "fr": {
       "accuracy": 0.875,
       "cc": 83.54690692562396,
       "cc_by_type": {
        "choice": 79.59183673469389,
        "noul": 100,
        "score": 71.04888404217799
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 83.54690692562396
      },
      "hi": {
       "accuracy": 0.6486486486486487,
       "cc": 46.51976370467721,
       "cc_by_type": {
        "choice": 58.677685950413235,
        "noul": 33.33333333333333,
        "score": 47.54827183028504
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 46.51976370467721
      },
      "id": {
       "accuracy": 0.926829268292683,
       "cc": 86.63403495463008,
       "cc_by_type": {
        "choice": 72.35023041474655,
        "noul": 100,
        "score": 87.55187444914372
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 86.63403495463008
      },
      "it": {
       "accuracy": 0.8444444444444444,
       "cc": 71.5725840321004,
       "cc_by_type": {
        "choice": 76.04259094942324,
        "noul": 66.66666666666667,
        "score": 72.00849448021127
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 71.5725840321004
      },
      "ja": {
       "accuracy": 0.8181818181818182,
       "cc": 76.40320576049011,
       "cc_by_type": {
        "choice": 53.36787564766839,
        "noul": 100,
        "score": 75.84174163380194
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 76.40320576049011
      },
      "ko": {
       "accuracy": 0.7209302325581395,
       "cc": 62.65982563130811,
       "cc_by_type": {
        "choice": 55.88235294117647,
        "noul": 60.00000000000001,
        "score": 72.09712395274785
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 62.65982563130811
      },
      "mixed": {
       "accuracy": 0.96875,
       "cc": 92.86911716343832,
       "cc_by_type": {
        "choice": 94.5994599459946,
        "noul": 89.99999999999999,
        "score": 94.00789154432036
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 92.86911716343832
      },
      "nl": {
       "accuracy": 0.75,
       "cc": 50.3321571323109,
       "cc_by_type": {
        "choice": 64.62264150943396,
        "noul": 80,
        "score": 6.37382988749875
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 50.3321571323109
      },
      "no": {
       "accuracy": 0.7647058823529411,
       "cc": 65.7551280522715,
       "cc_by_type": {
        "choice": 53.52839931153184,
        "noul": 86.66666666666667,
        "score": 57.070318178616
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 65.7551280522715
      },
      "pl": {
       "accuracy": 0.7719298245614035,
       "cc": 65.42784170752437,
       "cc_by_type": {
        "choice": 56.891963810537526,
        "noul": 100,
        "score": 39.39156131203556
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 65.42784170752437
      },
      "pt": {
       "accuracy": 0.7872340425531915,
       "cc": 75.09301738769823,
       "cc_by_type": {
        "choice": 58.06988352745424,
        "noul": 100,
        "score": 67.20916863564042
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 75.09301738769823
      },
      "sv": {
       "accuracy": 0.782608695652174,
       "cc": 66.09098036188544,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 65.12582749161533
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 66.09098036188544
      },
      "tr": {
       "accuracy": 0.8478260869565217,
       "cc": 84.2422138453902,
       "cc_by_type": {
        "choice": 58.950969213226905,
        "noul": 100,
        "score": 93.77567232294366
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 84.2422138453902
      },
      "uk": {
       "accuracy": 0.8392857142857143,
       "cc": 77.52228614905495,
       "cc_by_type": {
        "choice": 67.12328767123289,
        "noul": 100,
        "score": 65.44357077593193
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 77.52228614905495
      },
      "zh": {
       "accuracy": 0.8780487804878049,
       "cc": 83.29410729446424,
       "cc_by_type": {
        "choice": 81.40495867768595,
        "noul": 100,
        "score": 68.47736320570674
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 83.29410729446424
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7666666666666667,
       "cc": 68.17249497327498,
       "cc_by_type": {
        "choice": 68.17249497327498
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 68.17249497327498
      },
      "noul": {
       "accuracy": 0.9066666666666666,
       "cc": 81.33333333333333,
       "cc_by_type": {
        "noul": 81.33333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 81.33333333333333
      },
      "score": {
       "accuracy": 0.8186666666666667,
       "cc": 71.1620723673397,
       "cc_by_type": {
        "score": 71.1620723673397
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 71.1620723673397
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.8125,
       "cc": 76.8043323218334,
       "cc_by_type": {
        "choice": 64.41808747220162,
        "noul": 71.42857142857142,
        "score": 94.56633806472719
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 76.8043323218334
      },
      "code_linting": {
       "accuracy": 0.8405797101449275,
       "cc": 72.85323443902551,
       "cc_by_type": {
        "choice": 85.30312308634416,
        "noul": 83.33333333333333,
        "score": 49.92324689739903
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 72.85323443902551
      },
      "customer_support": {
       "accuracy": 0.8787878787878788,
       "cc": 78.1828059952545,
       "cc_by_type": {
        "choice": 81.54981549815498,
        "noul": 60.00000000000001,
        "score": 92.99860248760852
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 78.1828059952545
      },
      "demand_forecasting": {
       "accuracy": 0.5606060606060606,
       "cc": 39.25352471217226,
       "cc_by_type": {
        "choice": 18.119787717968165,
        "noul": 38.46153846153846,
        "score": 61.17924795701013
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 39.25352471217226
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8225806451612904,
       "cc": 74.404448945862,
       "cc_by_type": {
        "choice": 70.44334975369458,
        "noul": 88.23529411764706,
        "score": 64.53470296624437
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 74.404448945862
      },
      "feature_extraction": {
       "accuracy": 0.7777777777777778,
       "cc": 66.54767650219337,
       "cc_by_type": {
        "choice": 58.54271356783919,
        "noul": 71.42857142857142,
        "score": 69.67174451016949
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 66.54767650219337
      },
      "financial_crime": {
       "accuracy": 0.7777777777777778,
       "cc": 58.573354549119635,
       "cc_by_type": {
        "choice": 71.29186602870813,
        "noul": 45.45454545454546,
        "score": 58.97365216410532
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 58.573354549119635
      },
      "gaming": {
       "accuracy": 0.7931034482758621,
       "cc": 75.2478079578675,
       "cc_by_type": {
        "choice": 52.19123505976096,
        "noul": 100,
        "score": 73.55218881384154
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 75.2478079578675
      },
      "insurance_claims": {
       "accuracy": 0.7741935483870968,
       "cc": 62.695259081922266,
       "cc_by_type": {
        "choice": 55.45657015590201,
        "noul": 76.47058823529412,
        "score": 56.158618854570676
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 62.695259081922266
      },
      "knowledge_graphs": {
       "accuracy": 0.7910447761194029,
       "cc": 71.63734290669588,
       "cc_by_type": {
        "choice": 73.01349325337331,
        "noul": 100,
        "score": 41.89853546671433
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 71.63734290669588
      },
      "lead_generation": {
       "accuracy": 0.8133333333333334,
       "cc": 55.16023439880013,
       "cc_by_type": {
        "choice": 66.14950634696756,
        "noul": 82.85714285714285,
        "score": 16.474053992289996
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 55.16023439880013
      },
      "legal_compliance": {
       "accuracy": 0.8392857142857143,
       "cc": 81.61560229422075,
       "cc_by_type": {
        "choice": 75.92778335005015,
        "noul": 100,
        "score": 68.91902353261207
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 81.61560229422075
      },
      "llm_guardrails": {
       "accuracy": 0.9154929577464789,
       "cc": 84.86967354903356,
       "cc_by_type": {
        "choice": 87.78004073319754,
        "noul": 92.3076923076923,
        "score": 74.52128760621083
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 84.86967354903356
      },
      "model_routing": {
       "accuracy": 0.831081081081081,
       "cc": 71.34907633753097,
       "cc_by_type": {
        "choice": 78.18937164618316,
        "noul": 88.23529411764706,
        "score": 47.622563248762695
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 71.34907633753097
      },
      "moderation_trust_safety": {
       "accuracy": 0.847457627118644,
       "cc": 76.62196485152516,
       "cc_by_type": {
        "choice": 73.8562091503268,
        "noul": 76.47058823529412,
        "score": 79.53909716895456
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 76.62196485152516
      },
      "other": {
       "accuracy": 0.65625,
       "cc": 56.15117186313163,
       "cc_by_type": {
        "choice": 40.06659267480577,
        "noul": 66.66666666666667,
        "score": 61.720256247922435
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 56.15117186313163
      },
      "recruiting": {
       "accuracy": 0.8461538461538461,
       "cc": 76.20292693779872,
       "cc_by_type": {
        "choice": 65.39293439077144,
        "noul": 100,
        "score": 63.21584642262474
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 76.20292693779872
      },
      "risk_assessment": {
       "accuracy": 0.8333333333333334,
       "cc": 76.1280304011235,
       "cc_by_type": {
        "choice": 56.06694560669455,
        "noul": 84.61538461538463,
        "score": 87.70176098129127
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 76.1280304011235
      },
      "scientific_discovery": {
       "accuracy": 0.8095238095238095,
       "cc": 71.06055273427346,
       "cc_by_type": {
        "choice": 55.8282208588957,
        "noul": 80,
        "score": 77.35343734392465
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 71.06055273427346
      },
      "search_retrieval": {
       "accuracy": 0.9701492537313433,
       "cc": 90.9539156006108,
       "cc_by_type": {
        "choice": 90.55118110236221,
        "noul": 85.71428571428572,
        "score": 96.59627998518448
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 90.9539156006108
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.37934705910513,
      92.39059270239676
     ],
     "I": [
      62.172608632114404,
      72.41523796417187
     ],
     "capability": [
      75.24254727854819,
      81.6043946418471
     ],
     "composite": null,
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.10060876207879,
     "capability": 79.63511459799219,
     "intelligence": 68.16962043390559
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7666666666666667,
      "calibration": 89.18434274970146,
      "cc": 68.17249497327498,
      "cc_by_type": {
       "choice": 68.17249497327498
      },
      "cc_tiered": 69.02336647508682,
      "cc_tiered_open": 69.12190578377826,
      "cc_tiered_sealed": 68.92482716639536,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 68.17249497327498
     },
     "noul": {
      "accuracy": 0.9066666666666666,
      "calibration": 93.40578444489012,
      "cc": 81.33333333333333,
      "cc_by_type": {
       "noul": 81.33333333333333
      },
      "cc_tiered": 76.52654959153057,
      "cc_tiered_open": 73.59903381642512,
      "cc_tiered_sealed": 79.45406536663602,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 81.33333333333333
     },
     "score": {
      "accuracy": 0.8186666666666667,
      "calibration": 90.71169909164482,
      "cc": 71.1620723673397,
      "cc_by_type": {
       "score": 71.1620723673397
      },
      "cc_tiered": 58.958945235099385,
      "cc_tiered_open": 47.98576057229005,
      "cc_tiered_sealed": 69.93212989790872,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 71.1620723673397
     }
    },
    "raw_on_A": {
     "calibration": 90.6829365031639,
     "capability": 78.48667295933309,
     "intelligence": 66.29040941550228
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 75.33265760126677,
    "Emphasis on Accuracy 60:20:20": 72.29408857226244,
    "Emphasis on Cost 20:20:60": 77.38514304536655,
    "Emphasis on Speed 20:60:20": 76.51929431529942,
    "Intelligence only": 68.16962043390559
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "tattrongvu (Hugging Face)",
   "axes": {
    "calibration": 91.73503889029561,
    "cost": 51.40971770392302,
    "intelligence": 71.63230144667592,
    "speed": 84.65261351124272
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.028876069431341744,
      "ece_score": 94.22478611373165,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 92.96845563221524,
      "score_before_time_limit": 94.22478611373165,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.1253333333333333,
      "brier": 0.07307799064054041,
      "ece": 0.04494305379107265,
      "n": 375,
      "score": 90.52599516582929,
      "score_before_time_limit": 91.01138924178547,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9386666666666666,
      "interval90_width": 0.17200000000000007,
      "n": 375,
      "nrps": 0.04654031677107972,
      "score": 91.71066587284233,
      "score_before_time_limit": 92.45026801697816,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.05222716144467865
     }
    },
    "score": 91.73503889029561
   },
   "capability": 81.68367016848578,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 13044.9643397 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=native). The row's own scored cost basis: 1,334,761 input + 0 output tokens over 1,485 decisions = 898.828956229 tokens/decision. GPU-time cost USD 0.0416558572638797/1,000; old list-price cost USD 0.0808946060606061/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0416558572638797/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.15155237999999996
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0416558572638797
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jeffan-31b-1710",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5741917808219177/h recorded by the runner. One VM for both rows back to back, 31b then 4b. Engine and loopback shim in a --network none --read-only container per row; serial requests, read-only code/model/input, blank credential variables, HF offline flags. In-container network cut proven before sealed input: only loopback, no default route, DNS and TCP fail; code inventory/hashes and model files checked. Attempt 2 measured 24 Oct 2026 07:59-08:36Z; this row's measurement container 1035 s, run cap 1945 s, complete=true, deadline_hit=false, rc 0; scored loop 862.812179 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 5f578c036684557641cbfa55bfa043ed209775cbf793d05c4f78c1f4ce7c4fe0. Row cleanup verified; final exact-id VM teardown confirms provider absent and guard closed at 08:36:06Z. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.99205442745344,
    "I_sealed": 70.27254846589841,
    "base": 71.63230144667592,
    "excess": -7.120020808800682,
    "gap": 2.719505961555029,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 78.4827278527896,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 83.01352626612143,
       "judge": 72.31377719182598,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 69.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 70.54519526393928,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 66.47787147888829,
       "hard": 77.29517951037028,
       "judge": 75.12442806811568,
       "standard": 52.21003945733818
      }
     },
     "sealed|choice": {
      "cc": 84.85434539262528,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 83.34682548860874,
       "judge": 85.01469147894223,
       "standard": 82.91499683611053
      }
     },
     "sealed|noul": {
      "cc": 60.194263790764055,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 49.09090909090909,
       "judge": 60.00000000000001,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 65.76903621430593,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 84.39945205262151,
       "hard": 64.68166177839782,
       "judge": 62.22278902181544,
       "standard": 63.94794795570004
      }
     }
    }
   },
   "jevbench_score": 71.26859587879677,
   "key": "jeffan-31b-1710",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 at the measured pin (model card and LICENSE); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "tattrongvu/Jeffan-31b-1710@cebfa86c2bed2b5c83b12e44de4374b20c87fd79",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9447852760736196,
    "decisive": 326,
    "decisive_rate": 0.8739946380697051,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 65,
   "ranked": true,
   "ranks": {
    "A": 65,
    "B": 41,
    "C": 51,
    "capability": 24
   },
   "repo": "https://huggingface.co/tattrongvu/Jeffan-31b-1710",
   "scores": {
    "A": 71.26859587879677,
    "B": 71.34104122366863,
    "C": 71.26859587879677
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.39501961600026336,
    "p50_s_raw": 0.12250980800013167,
    "p95_s_adjusted": 0.8672014321001599,
    "p95_s_raw": 0.3586007160500799
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN DECISION INDEX ENGINE, run unmodified: tattrongvu/Jeffan-31b-1710 at cebfa86c2bed2b5c83b12e44de4374b20c87fd79, three author package files (__init__.py, engine.py, render.py), the three DI-kit base files at f8552a13b643d39f61185f5951bb5fed16bedf51 and our transport-only jeffan_shim.py. Exact ten-file code manifest sha256 f38f38d69571180283539bb9d8fad4412a6376c5d010bbe53919f8a81a5b8004, hash-checked in the container. dm_decision_engine.DecisionEngine(max_len=98304, max_batched_tokens=4096, reread_below=0.9, gpu_mem=0.85), vLLM 0.29.0 bf16, prefix caching, causal attention; conditional read-twice when first-read top-1 probability is below 0.9 and the doubled prompt fits. The predeclared memory ladder was not used. Python 3.12.3, torch 2.13.0+cu130, transformers 5.20.0, tokenizers 0.23.3, huggingface-hub 1.33.0. The author verified this stack on B200 only; this measures RTX PRO 6000. Base image vllm/vllm-openai:v0.29.0@sha256:c2914767605584b6d8f45686b82de173ecc99e781897aa3d0a66dacd72c51ae1; derived image sha256:b911dd6884706894279758c428e1a8884d4b995371fc4cf09c89136ee825fffd, hash-pinned wheel overlay. The submitted code was reviewed and cleared at the exact pins with the reviewed safeguards; evaluator exposure and the device were cleared for both attempts. Source review records dormant Hub-capable hf_hub_download/from_pretrained paths; the model argument is a hash-checked local directory, revision=None, offline flags set and network disabled. No author shell/subprocess, eval/exec, pickle or trust_remote_code path on the reviewed serving path. The 4b vLLM plugin was excluded and never executed. The runner and device selection change were reviewed; the amended runner was rechecked with the suite rerun and amendment recorded before retry. Weights: Apache-2.0 at this pin (earlier CC BY-NC 4.0 pins are not measured), public, ungated; bf16, 31,273,086,512 parameters, 62,546,338,264 weight bytes.  FIRST MEASUREMENT of Jeffan-31b-1710 (our own intake). AUTHOR'S OWN ENGINE on our rented hardware, one request at a time; our harness replaces the author's Decision Index pipeline. NATIVE OUTPUTS: next-token probabilities over the option codes, choice argmax and probabilities, noul P(true), score expectation and probabilities; the first-party transport shim adds no prompt text or probability arithmetic. AUTHOR EXPOSURE: the preregistration records training data as undisclosed (Decision Index note) and author-reported Decision Index suite figures; no overlap or contamination audit was run by us. The author's reported figures are not independently verified by us. CONTEXT: engine max_len 98,304 prompt tokens; no lower evaluator-imposed cap and no truncation. All 1,500 items answered HTTP 200, 0 refusals; longest first read 81,715 tokens. The card states inputs up to 8,192 tokens and the author validated at most 32,362 first-read tokens; these long items exceed that validation. 10 s RULE: exactly 15 decisions over 10 s, the standard long items at positions in the pre-registered long-item set; these count wrong in the v1.7 share view (frozen v1.6 and v1.7-off views apply no time limit). Maximum 44.157234 s, median 0.122518 s, p95 0.358708 s. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. ATTEMPT HISTORY: attempt 1, pod window 24 Oct 2026 05:52-06:21Z (about USD 1.24) on a separate disposable GPU instance, stopped the 31b row after 1,057 of 1,500 items at the runner's own time cap. The smoke under-predicted long-item latency because repeated probe prompts hit the prefix cache. The partial is retained as a receipt only, never scored or merged; the 4b row had not started. Amendment 3 corrected the first-party runner to use cold nonce-bearing probes, maximum anchor latency and a long-item budget floor; a further review preceded the retry. evaluator exposure and device ruling admitted the fresh full 31b run as the final same-use retry; 4b remained a first use. Attempt 2 is this measurement, both rows back to back on new a separate disposable GPU instance, 24 Oct 07:59-08:36Z, about USD 1.57 rental. Between attempts only our first-party runner (smoke probes and fit gate, pre-registration Amendment 3) changed; the scoring harness, model, author code, input and engine settings are unchanged, and the pinned image stack was rebuilt on the new VM. Over the cost cap under the earlier list-price basis; inside both caps from v1.8.0 (GPU-time cost basis).",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 90.78364769739426,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 72.3509430921828
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 90.78364769739426
      },
      "calibration_gen": {
       "accuracy": 0.8118811881188119,
       "cc": 70.4990841343402,
       "cc_by_type": {
        "choice": 84.09331919406152,
        "noul": 52,
        "score": 75.40393320895905
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 70.4990841343402
      },
      "composite_gen": {
       "accuracy": 0.9108910891089109,
       "cc": 85.83224203573293,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 77.77777777777777,
        "score": 82.97098084974627
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.83224203573293
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.78298898647171,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 97.34896695941512
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.78298898647171
      },
      "jaggedness": {
       "accuracy": 0.8,
       "cc": 65.21005041341623,
       "cc_by_type": {
        "choice": 78.34456207892205,
        "noul": 40.54054054054055,
        "score": 76.7450486207861
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 65.21005041341623
      },
      "llm_router": {
       "accuracy": 0.9041533546325878,
       "cc": 80.6688597599484,
       "cc_by_type": {
        "choice": 90.49044642070969,
        "noul": 93.81443298969072,
        "score": 57.70169986944481
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 80.6688597599484
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 49.27195296946737,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 25,
        "score": 47.5581269496392
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 49.27195296946737
      },
      "multilingual": {
       "accuracy": 0.7709090909090909,
       "cc": 59.75363667787247,
       "cc_by_type": {
        "choice": 79.45875448320835,
        "noul": 36.111111111111114,
        "score": 63.69104443929796
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 59.75363667787247
      },
      "negation_pair_gen": {
       "accuracy": 0.8837209302325582,
       "cc": 76.39048614552513,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 73.33333333333334,
        "score": 62.50479176990871
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 76.39048614552513
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 86.47973622784764,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 71.4512206955549
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 86.47973622784764
      },
      "usecase": {
       "accuracy": 0.7909090909090909,
       "cc": 68.50358527761284,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 62.5,
        "score": 66.2950695583287
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 68.50358527761284
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8125,
       "cc": 57.917196734831215,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": -7.200790747887287
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 57.917196734831215
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 77.78630484073538,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 33.35891452220612
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 77.78630484073538
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 76.66453093859816,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 78.88739812552899
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 76.66453093859816
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.58604847392802,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.94436516107639
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.58604847392802
      },
      "en": {
       "accuracy": 0.8549087749782798,
       "cc": 75.53842125405133,
       "cc_by_type": {
        "choice": 85.50176346395575,
        "noul": 70.50359712230217,
        "score": 70.60990317589604
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 75.53842125405133
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 71.24640609810298,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 23.52714651616864
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.24640609810298
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.5130044521433,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 98.99256526899825
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.5130044521433
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 88.65781563933801,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 87.14991750624928
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 88.65781563933801
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 76.43656310769823,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 50,
        "score": 79.30968932309472
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 76.43656310769823
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 68.9658307236104,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 20.927110253450067
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 68.9658307236104
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 44.59217191994967,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 13.776515759849017
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 44.59217191994967
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 12.72898872647921,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 82.63141062388209
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 12.72898872647921
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.96095473924714,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 94.54953088440809
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.96095473924714
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -14.965639046318811,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 7.051134809095516
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 10.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 10.483870967741936
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 67.40120233664327,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 69.82104247332381
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 67.40120233664327
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 58.32982292531735,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 57.417282564991545
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 58.32982292531735
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 48.6264307338863,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 19.999999999999996,
        "score": 51.41120709527591
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 48.6264307338863
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 70.95545468890508,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 12.866364066715242
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 70.95545468890508
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 80.94549529754356,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 75.09455040875973
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 80.94549529754356
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 38.564498140494244,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 49.02682775481608
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 38.564498140494244
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8786666666666667,
       "cc": 84.32805832488184,
       "cc_by_type": {
        "choice": 84.32805832488184
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.32805832488184
      },
      "noul": {
       "accuracy": 0.8213333333333334,
       "cc": 64.26666666666667,
       "cc_by_type": {
        "noul": 64.26666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 64.26666666666667
      },
      "score": {
       "accuracy": 0.7733333333333333,
       "cc": 67.67374817243945,
       "cc_by_type": {
        "score": 67.67374817243945
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 67.67374817243945
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 33.21690881679642,
       "cc_by_type": {
        "choice": 68,
        "noul": 0,
        "score": 31.650726450389254
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 33.21690881679642
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 84.32387200910605,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 71.60515639998897
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 84.32387200910605
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 29.066115138365063,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50,
        "score": 2.2389958215992434
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 29.066115138365063
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -3.1361152440208784,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -33.333333333333336,
        "score": -17.595480234986603
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 35.210332375925105,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 67.94983770748546
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 35.210332375925105
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 33.63873931643062,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 33.33333333333333,
        "score": 5.318733672562326
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 33.63873931643062
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 53.405528031692114,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 14.28571428571428,
        "score": 55.7347913779895
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 53.405528031692114
      },
      "gaming": {
       "accuracy": 0.8181818181818182,
       "cc": 66.48399365709115,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 50,
        "score": 70.35790431622114
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 66.48399365709115
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 38.547249746550726,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 58.141749239652164
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 38.547249746550726
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 81.83766752199955,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100,
        "score": 58.360968304756675
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 81.83766752199955
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 64.59063160789553,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 27.62202402265299
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 64.59063160789553
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 12.068318831643523,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -0.35418329001566384
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 12.068318831643523
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 64.85792290166587,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 74.57376870499763
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 64.85792290166587
      },
      "model_routing": {
       "accuracy": 0.8755760368663594,
       "cc": 76.00451100954336,
       "cc_by_type": {
        "choice": 88.31168831168831,
        "noul": 93.93939393939394,
        "score": 45.762450777547855
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 76.00451100954336
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 24.165403800315858,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 84.0125261802182
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 24.165403800315858
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 62.22189994439916,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 86.75855034387531
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 62.22189994439916
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 42.19246157222563,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 50,
        "score": 20.912360086135028
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 42.19246157222563
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 44.21569095834614,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 0,
        "score": 61.63258012141524
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 44.21569095834614
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.90957867741692,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.72873603225078
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.90957867741692
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.62696026019485,
      92.3898122960135
     ],
     "I": [
      66.74809318659113,
      75.34912722976662
     ],
     "capability": [
      77.77844533915831,
      83.22563961157051
     ],
     "composite": null,
     "seed": 36
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.73503889029561,
     "capability": 81.68367016848578,
     "intelligence": 71.63230144667592
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8786666666666667,
      "calibration": 92.96845563221524,
      "cc": 84.32805832488184,
      "cc_by_type": {
       "choice": 84.32805832488184
      },
      "cc_tiered": 81.66853662270744,
      "cc_tiered_open": 78.4827278527896,
      "cc_tiered_sealed": 84.85434539262528,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.32805832488184
     },
     "noul": {
      "accuracy": 0.8213333333333334,
      "calibration": 90.52599516582929,
      "cc": 64.26666666666667,
      "cc_by_type": {
       "noul": 64.26666666666667
      },
      "cc_tiered": 65.07125197819776,
      "cc_tiered_open": 69.94824016563147,
      "cc_tiered_sealed": 60.194263790764055,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 64.26666666666667
     },
     "score": {
      "accuracy": 0.7733333333333333,
      "calibration": 91.71066587284233,
      "cc": 67.67374817243945,
      "cc_by_type": {
       "score": 67.67374817243945
      },
      "cc_tiered": 68.1571157391226,
      "cc_tiered_open": 70.54519526393928,
      "cc_tiered_sealed": 65.76903621430593,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 67.67374817243945
     }
    },
    "raw_on_A": {
     "calibration": 88.93954523997444,
     "capability": 80.14320664914894,
     "intelligence": 71.34686805832344
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.33536977605696,
    "Emphasis on Accuracy 60:20:20": 68.35727214368964,
    "Emphasis on Cost 20:20:60": 59.43332014361794,
    "Emphasis on Speed 20:60:20": 72.6208913284571,
    "Intelligence only": 71.63230144667592
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "tattrongvu (Hugging Face)",
   "axes": {
    "calibration": 87.9034951152282,
    "cost": 78.67753572713382,
    "intelligence": 40.09995458474205,
    "speed": 93.04878911983016
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04890108419083565,
      "ece_score": 90.21978316183286,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 90.21978316183286,
      "score_before_time_limit": 90.21978316183286,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3699731903485255,
      "brier": 0.14739969780221118,
      "ece": 0.08497512533164059,
      "n": 373,
      "score": 83.00497493367189,
      "score_before_time_limit": 83.00497493367189,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9543010752688172,
      "interval90_width": 0.4735663082437274,
      "n": 372,
      "nrps": 0.09846026008120294,
      "score": 90.48572725017985,
      "score_before_time_limit": 90.48572725017985,
      "time_limit_failed_share": 0,
      "top_ece": 0.04591259745760014
     }
    },
    "score": 87.9034951152282
   },
   "capability": 64.00172484998512,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 776,399 input + 0 output tokens over 1,485 decisions = 522.827609428 tokens/decision. GPU-time cost USD 0.00513746420646507/1,000; old list-price cost USD 0.0156848282828283/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00513746420646507/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.01552798
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005137464206465065
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jeffan-4B-2010",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5741917808219177/h recorded by the runner. One VM for both rows back to back, 31b then 4b. Engine and loopback shim in a --network none --read-only container per row; serial requests, read-only code/model/input, blank credential variables, HF offline flags. In-container network cut proven before sealed input: only loopback, no default route, DNS and TCP fail; code inventory/hashes and model files checked. Attempt 2 measured 24 Oct 2026 07:59-08:36Z; this row's measurement container 94 s, run cap 183 s, complete=true, deadline_hit=false, rc 0; scored loop 50.604333 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 e32cbcf090bccf4750a0225b30d9cd73aa0636be787f9f517d3e97946691121a. Row cleanup verified; final exact-id VM teardown confirms provider absent and guard closed at 08:36:06Z. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 47.662298511076926,
    "I_sealed": 32.537610658407175,
    "base": 40.09995458474205,
    "excess": 5.2851610823140405,
    "gap": 15.124687852669751,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 72.15914308499305,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 58.47750865051904,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 31.68391994478951,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": 0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 39.14383250344822,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.06727265030784,
       "hard": 41.445405969054626,
       "judge": 42.37827155697271,
       "standard": 19.22730691851886
      }
     },
     "sealed|choice": {
      "cc": 62.82197435013218,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 80.94027954256671,
       "hard": 58.36706372152191,
       "judge": 59.4515181194907,
       "standard": 67.72832735709765
      }
     },
     "sealed|noul": {
      "cc": 1.5438817487196292,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 34.48275862068966,
       "hard": 9.090909090909083,
       "judge": -35.38461538461538,
       "standard": 25.373134328358216
      }
     },
     "sealed|score": {
      "cc": 33.24697587636973,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 56.57048051927891,
       "hard": 29.398575584954646,
       "judge": 30.460231180970943,
       "standard": 33.462141180843474
      }
     }
    }
   },
   "jevbench_score": 43.04448867574887,
   "key": "jeffan-4b-2010",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 at the measured pin (model card and LICENSE); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "tattrongvu/Jeffan-4B-2010@937a008f86f4a20a903d628f22ffad3e3f431d2c",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9063829787234042,
    "decisive": 235,
    "decisive_rate": 0.6300268096514745,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 140,
   "ranked": true,
   "ranks": {
    "A": 140,
    "B": 147,
    "C": 145,
    "capability": 136
   },
   "repo": "https://huggingface.co/tattrongvu/Jeffan-4B-2010",
   "scores": {
    "A": 43.04448867574887,
    "B": 37.965581433007344,
    "C": 29.892006024825612
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20658003700000335,
    "p50_s_raw": 0.02829001850000168,
    "p95_s_adjusted": 0.23990137499999947,
    "p95_s_raw": 0.04495068749999973
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN DECISION INDEX ENGINE, run unmodified: tattrongvu/Jeffan-4B-2010 at 937a008f86f4a20a903d628f22ffad3e3f431d2c, three author package files (__init__.py, engine.py, render.py), the three DI-kit base files at f8552a13b643d39f61185f5951bb5fed16bedf51 and our transport-only jeffan_shim.py. Exact ten-file code manifest sha256 f38f38d69571180283539bb9d8fad4412a6376c5d010bbe53919f8a81a5b8004, hash-checked in the container. jeffan_engine.DecisionEngine(max_len=65536, budget=65536, device=\"cuda\"), Transformers bf16, SDPA with explicit bidirectional masks for full-attention layers and causal Gated-DeltaNet layers, one read. Python 3.12.3, torch 2.13.0+cu130 (author note: 2.14.1+cu130; pyproject requires >=2.9), transformers 5.17.0, tokenizers 0.23.3, flash-linear-attention 0.5.2, fla-core 0.5.2, huggingface-hub 1.33.0. Optional causal-conv1d absent: torch fallback. vLLM is installed in the image but is not the 4b serving backend. Base image vllm/vllm-openai:v0.29.0@sha256:c2914767605584b6d8f45686b82de173ecc99e781897aa3d0a66dacd72c51ae1; derived image sha256:4f4d8a799589b6137954bd2fd7d0f2c4ac8b126ad08e5d31d261efe409bd4b51, hash-pinned wheel overlay. The submitted code was reviewed and cleared at the exact pins with the reviewed safeguards; evaluator exposure and the device were cleared for both attempts. Source review records dormant Hub-capable hf_hub_download/from_pretrained paths; the model argument is a hash-checked local directory, revision=None, offline flags set and network disabled. No author shell/subprocess, eval/exec, pickle or trust_remote_code path on the reviewed serving path. The 4b vLLM plugin was excluded and never executed. The runner and device selection change were reviewed; the amended runner was rechecked with the suite rerun and amendment recorded before retry. Weights: Apache-2.0 at this pin, public, ungated; bf16, 4,205,751,296 parameters, 8,411,558,400 weight bytes.  FIRST MEASUREMENT of Jeffan-4B-2010 (our own intake). AUTHOR'S OWN ENGINE on our rented hardware, one request at a time; our harness replaces the author's Decision Index pipeline. NATIVE OUTPUTS: next-token probabilities over the option codes, choice argmax and probabilities, noul P(true), score expectation and probabilities; the first-party transport shim adds no prompt text or probability arithmetic. AUTHOR EXPOSURE: the preregistration records training data as undisclosed (Decision Index note) and author-reported Decision Index suite figures; no overlap or contamination audit was run by us. The author's reported figures are not independently verified by us. NOT ANSWERED, counted wrong: 15 items exceed the author engine's 65,536-token max_len/budget, HTTP 422 Unsupported, exactly the pre-registered standard long positions in the pre-registered long-item set; no truncation. Answered 1,485 of 1,500 items HTTP 200; no unexpected refusal. 10 s RULE: 0 decisions over 10 s; no additional decision is counted wrong by that rule in the v1.7 share view (frozen v1.6 and v1.7-off views apply no time limit). Maximum 0.083058 s, median 0.028291 s, p95 0.044957 s. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. ATTEMPT HISTORY: attempt 1, pod window 24 Oct 2026 05:52-06:21Z (about USD 1.24) on a separate disposable GPU instance, stopped the 31b row after 1,057 of 1,500 items at the runner's own time cap. The smoke under-predicted long-item latency because repeated probe prompts hit the prefix cache. The partial is retained as a receipt only, never scored or merged; the 4b row had not started. Amendment 3 corrected the first-party runner to use cold nonce-bearing probes, maximum anchor latency and a long-item budget floor; a further review preceded the retry. evaluator exposure and device ruling admitted the fresh full 31b run as the final same-use retry; 4b remained a first use. Attempt 2 is this measurement, both rows back to back on new a separate disposable GPU instance, 24 Oct 07:59-08:36Z, about USD 1.57 rental. Between attempts only our first-party runner (smoke probes and fit gate, pre-registration Amendment 3) changed; the scoring harness, model, author code, input and engine settings are unchanged, and the pinned image stack was rebuilt on the new VM. ",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5909090909090909,
       "cc": 17.107216000768457,
       "cc_by_type": {
        "choice": 50,
        "noul": 50,
        "score": -48.678351997694634
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 17.107216000768457
      },
      "calibration_gen": {
       "accuracy": 0.37623762376237624,
       "cc": 7.562217511295402,
       "cc_by_type": {
        "choice": 30.010604453870634,
        "noul": -28.000000000000004,
        "score": 20.676048080015576
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 7.562217511295402
      },
      "composite_gen": {
       "accuracy": 0.7029702970297029,
       "cc": 41.6822368932725,
       "cc_by_type": {
        "choice": 77.23577235772358,
        "noul": -3.703703703703709,
        "score": 51.514642025797656
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 41.6822368932725
      },
      "fairness": {
       "accuracy": 0.8333333333333334,
       "cc": 74.29181378692071,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 62.87544136076213
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 74.29181378692071
      },
      "jaggedness": {
       "accuracy": 0.49444444444444446,
       "cc": 12.959890259411873,
       "cc_by_type": {
        "choice": 48.0269489894129,
        "noul": -35.13513513513513,
        "score": 25.987856923957853
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 12.959890259411873
      },
      "llm_router": {
       "accuracy": 0.7859424920127795,
       "cc": 63.12199499130963,
       "cc_by_type": {
        "choice": 73.05626485867745,
        "noul": 67.01030927835052,
        "score": 49.29941083690091
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 63.12199499130963
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4444444444444444,
       "cc": -12.627343185750314,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -75,
        "score": -13.397493474776724
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6254545454545455,
       "cc": 34.063450895608206,
       "cc_by_type": {
        "choice": 62.83012716009129,
        "noul": -2.777777777777779,
        "score": 42.138003304511116
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 34.063450895608206
      },
      "negation_pair_gen": {
       "accuracy": 0.6744186046511628,
       "cc": 28.994252986539696,
       "cc_by_type": {
        "choice": 86.66666666666667,
        "noul": -6.666666666666665,
        "score": 6.982758959619084
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 28.994252986539696
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 85.81723374265215,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 69.46371323996847
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.81723374265215
      },
      "usecase": {
       "accuracy": 0.5681818181818182,
       "cc": 30.91043554539965,
       "cc_by_type": {
        "choice": 63.23529411764707,
        "noul": -4.1666666666666625,
        "score": 33.662679185218536
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 30.91043554539965
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": -1.2336293925260573,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0,
        "score": -46.558031034721026
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 43.53012068982637,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": -2.7429712638542103
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 43.53012068982637
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 62.97016399304912,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75,
        "score": 45.76889905879339
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 62.97016399304912
      },
      "el": {
       "accuracy": 0.4,
       "cc": 22.06279181925926,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.89414826969585
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 22.06279181925926
      },
      "en": {
       "accuracy": 0.6420503909643788,
       "cc": 38.65953483041661,
       "cc_by_type": {
        "choice": 65.5117706642584,
        "noul": 15.107913669064743,
        "score": 35.35892015792667
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 38.65953483041661
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 66.24333528280283,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 18.305862292127927
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.24333528280283
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 36.95274616427675,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 75.88556089720186
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 36.95274616427675
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 35.29290746824505,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 48.23166358120573
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 35.29290746824505
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 19.531242708314245,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -50,
        "score": 25.353504661255577
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 19.531242708314245
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 82.54417059540985,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 47.63251178622958
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 82.54417059540985
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 13.714413463902668,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 0,
        "score": 12.628546189960588
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 13.714413463902668
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -13.066722136521031,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 5.244278034881356
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.81697623873661,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 97.11759538287652
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.81697623873661
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 5.697940383667344,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 43.06784712502799
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 5.697940383667344
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -39.516129032258064,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 29.62098153412335,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -40,
        "score": 51.71525099255203
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 29.62098153412335
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 76.43182288942896,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 61.11632901006649
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 76.43182288942896
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 30.653224268712048,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 48.555417486987196
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 30.653224268712048
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 29.035136356137745,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50,
        "score": -19.416330062021547
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 29.035136356137745
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 42.508577211094895,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": -7.958139334457259
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 42.508577211094895
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 19.62220986732483,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -7.800037064692167
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 19.62220986732483
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7306666666666667,
       "cc": 65.21173386402343,
       "cc_by_type": {
        "choice": 65.21173386402343
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 65.21173386402343
      },
      "noul": {
       "accuracy": 0.568,
       "cc": 13.59999999999999,
       "cc_by_type": {
        "noul": 13.59999999999999
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 13.59999999999999
      },
      "score": {
       "accuracy": 0.5386666666666666,
       "cc": 35.71601417794966,
       "cc_by_type": {
        "score": 35.71601417794966
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 35.71601417794966
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -7.281748941189204,
       "cc_by_type": {
        "choice": 52,
        "noul": -75,
        "score": 1.1547531764323882
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 15.505884184811967,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -11.111111111111116,
        "score": 13.52938478355944
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 15.505884184811967
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 48.461591841203415,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 44.165263328488294
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 48.461591841203415
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -23.483617154781218,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -100,
        "score": -11.971319300600957
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": 9.948057697322461,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 21.148520918054338
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 9.948057697322461
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -35.472173838097454,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -33.333333333333336,
        "score": -110.19010641995273
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 32.3619825613593,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": -14.28571428571429,
        "score": 21.17558353841963
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 32.3619825613593
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -10.059986139273484,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100,
        "score": 32.53781161702275
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": -4.022713098007958,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -60,
        "score": 52.0985273726428
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 39.0492321797909,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 33.33333333333333,
        "score": 22.35826042231347
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 39.0492321797909
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 28.820093796807527,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 9.1992994782779
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 28.820093796807527
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -29.197088514175146,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100,
        "score": 9.182928005861656
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 43.44460100858145,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 19.999999999999996,
        "score": 33.36643258428559
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 43.44460100858145
      },
      "model_routing": {
       "accuracy": 0.8248847926267281,
       "cc": 66.28766302911839,
       "cc_by_type": {
        "choice": 83.63636363636363,
        "noul": 81.81818181818181,
        "score": 33.408443632809735
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.28766302911839
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 11.296360801445681,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 56.9217119628783
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 11.296360801445681
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 52.95878569800709,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 42.2561156826935
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 52.95878569800709
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": 2.8202015899512247,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50,
        "score": 17.573905262464507
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 2.8202015899512247
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 9.491140938992965,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 5.285017019877447
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 9.491140938992965
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 91.97693809933558,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 86.60697088163664
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 91.97693809933558
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.87939852911506,
      88.28583907929631
     ],
     "I": [
      34.57075435928627,
      44.161488911611954
     ],
     "capability": [
      59.16381168166789,
      65.51171574224394
     ],
     "composite": null,
     "seed": 37
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.9034951152282,
     "capability": 64.00172484998512,
     "intelligence": 40.09995458474205
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7306666666666667,
      "calibration": 90.21978316183286,
      "cc": 65.21173386402343,
      "cc_by_type": {
       "choice": 65.21173386402343
      },
      "cc_tiered": 67.4905587175626,
      "cc_tiered_open": 72.15914308499305,
      "cc_tiered_sealed": 62.82197435013218,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.21173386402343
     },
     "noul": {
      "accuracy": 0.568,
      "calibration": 83.00497493367189,
      "cc": 13.59999999999999,
      "cc_by_type": {
       "noul": 13.59999999999999
      },
      "cc_tiered": 16.613900846754568,
      "cc_tiered_open": 31.68391994478951,
      "cc_tiered_sealed": 1.5438817487196292,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 13.59999999999999
     },
     "score": {
      "accuracy": 0.5386666666666666,
      "calibration": 90.48572725017985,
      "cc": 35.71601417794966,
      "cc_by_type": {
       "score": 35.71601417794966
      },
      "cc_tiered": 36.19540418990898,
      "cc_tiered_open": 39.14383250344822,
      "cc_tiered_sealed": 33.24697587636973,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.71601417794966
     }
    },
    "raw_on_A": {
     "calibration": 85.75583127024964,
     "capability": 63.842376704635804,
     "intelligence": 41.928922139021964
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 39.87217498129024,
    "Emphasis on Accuracy 60:20:20": 32.72622182786017,
    "Emphasis on Cost 20:20:60": 43.56851777259423,
    "Emphasis on Speed 20:60:20": 46.016044721286285,
    "Intelligence only": 25.792392766272524
   }
  },
  {
   "adapter_id": "harness/specs/jev-1.13.0.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "TypeSafe AI",
   "axes": {
    "calibration": 90.63220203152369,
    "cost": 54.733842199536014,
    "intelligence": 63.62075784024779,
    "speed": 91.21186387733223
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03783700194123639,
      "ece_score": 92.43259961175272,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 92.43259961175272,
      "score_before_time_limit": 92.43259961175272,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.21293800539083563,
      "brier": 0.06638921832884097,
      "ece": 0.06361185983827498,
      "n": 371,
      "score": 87.277628032345,
      "score_before_time_limit": 87.277628032345,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9728997289972899,
      "interval90_width": 0.24954832881662145,
      "n": 369,
      "nrps": 0.05074999798842498,
      "score": 92.18637845047334,
      "score_before_time_limit": 92.18637845047334,
      "time_limit_failed_share": 0,
      "top_ece": 0.05276121650105413
     }
    },
    "score": 90.63220203152369
   },
   "capability": 77.12647993588574,
   "class": "jev",
   "composite_ci95": {
    "A": [
     69.09826862399638,
     72.35050977564978
    ],
    "B": [
     66.56025704419686,
     71.32565861066611
    ],
    "C": [
     64.31427798334369,
     72.35050977564978
    ]
   },
   "cost": {
    "basis": "TypeSafe list tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.031780532
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03227542180094787
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jev 1.13.0 (TypeSafe AI)",
   "endpoint_condition": "the operator's hosted API",
   "endpoint_kind": "api",
   "full_coverage": true,
   "gpu": null,
   "intelligence": {
    "I_open": 65.60411433522728,
    "I_sealed": 61.637401345268295,
    "base": 63.62075784024779,
    "excess": 1.3933253475207414,
    "gap": 3.9667129899589852,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 79.0195653974535,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 71.68921044353571,
       "judge": 84.17930125247197,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 54.46376811594203,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 47.82608695652173,
       "judge": 39.99999999999999,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 63.32900949228633,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 76.45454545454547,
       "hard": 65.44837261503929,
       "judge": 68.91525423728812,
       "standard": 44.148148148148145
      }
     },
     "sealed|choice": {
      "cc": 75.78213638772388,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 76.05498484960461,
       "hard": 73.02036199095022,
       "judge": 75.79143389199255,
       "standard": 81.15531469392782
      }
     },
     "sealed|noul": {
      "cc": 46.14871857614774,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 80,
       "hard": 28.155339805825253,
       "judge": 47.36842105263157,
       "standard": 63.38028169014085
      }
     },
     "sealed|score": {
      "cc": 62.981349071933266,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 81.82978723404256,
       "hard": 55.9193404634581,
       "judge": 71.57122119397567,
       "standard": 54.796339024765395
      }
     }
    }
   },
   "jevbench_score": 71.44711653093671,
   "key": "jev-1.13.0",
   "last_measured_on": "2026-10-05",
   "licence": "proprietary API",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "jev-1.13.0 (response model id on all 590 successful rows)",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.976027397260274,
    "decisive_rate": 0.7870619946091644,
    "supported": true,
    "valid": 371
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 12,
    "axes": {
     "calibration": 90.63220203152369,
     "cost": 54.733842199536014,
     "intelligence": 63.62075784024779,
     "speed": 91.49989299046987
    },
    "board_release": "v1.6.43",
    "capability": 77.12647993588574,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": 21,
    "composite_ci95": {
     "A": [
      69.1394876572671,
      72.39570149403289
     ],
     "B": [
      66.59085018153897,
      71.36079037075339
     ],
     "C": [
      64.35267606102977,
      72.39570149403289
     ]
    },
    "jevbench_score": 71.49118639284514,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 18,
    "ranks": {
     "A": 18,
     "B": 22,
     "C": 18,
     "capability": 38
    },
    "scores": {
     "A": 71.49118639284514,
     "B": 69.76507971323585,
     "C": 71.49118639284514
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.23910215869545937,
     "p50_s_raw": 0.23910215869545937,
     "p95_s_adjusted": 0.29609236158430574,
     "p95_s_raw": 0.29609236158430574
    },
    "views": {
     "Balanced 33:33:33": 66.78934094542959,
     "Emphasis on Accuracy 60:20:20": 65.48477335742311,
     "Emphasis on Cost 20:20:60": 61.38146766808138,
     "Emphasis on Speed 20:60:20": 74.8779907919869,
     "Intelligence only": 63.620757840247784
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": 64,
   "ranked": true,
   "ranks": {
    "A": 64,
    "B": 56,
    "C": 50,
    "capability": 51
   },
   "repo": "https://docs.typesafe.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.44711653093671,
    "B": 69.73150119125978,
    "C": 71.44711653093671
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24289856851100922,
    "p50_s_raw": 0.24289856851100922,
    "p95_s_adjusted": 0.31145023070275785,
    "p95_s_raw": 0.31145023070275785
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "closed",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.83738964744846,
      90.80253311928502
     ],
     "I": [
      57.864503927061655,
      67.65512541027901
     ],
     "capability": [
      72.76645552638956,
      78.36978036921771
     ],
     "composite": {
      "A": [
       69.09826862399638,
       72.35050977564978
      ],
      "B": [
       66.56025704419686,
       71.32565861066611
      ],
      "C": [
       64.31427798334369,
       72.35050977564978
      ]
     },
     "seed": 44
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.63220203152369,
     "capability": 77.12647993588574,
     "intelligence": 63.62075784024779
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8173333333333334,
      "calibration": 92.43259961175272,
      "cc": 76.63954226386139,
      "cc_by_type": {
       "choice": 76.63954226386139
      },
      "cc_tiered": 77.40085089258869,
      "cc_tiered_open": 79.0195653974535,
      "cc_tiered_sealed": 75.78213638772388,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.63954226386139
     },
     "noul": {
      "accuracy": 0.76,
      "calibration": 87.277628032345,
      "cc": 52,
      "cc_by_type": {
       "noul": 52
      },
      "cc_tiered": 50.30624334604489,
      "cc_tiered_open": 54.46376811594203,
      "cc_tiered_sealed": 46.14871857614774,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 52
     },
     "score": {
      "accuracy": 0.752,
      "calibration": 92.18637845047334,
      "cc": 63.113849016480586,
      "cc_by_type": {
       "score": 63.113849016480586
      },
      "cc_tiered": 63.1551792821098,
      "cc_tiered_open": 63.32900949228633,
      "cc_tiered_sealed": 62.981349071933266,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 63.113849016480586
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.73806378390636,
    "Emphasis on Accuracy 60:20:20": 65.45518786046593,
    "Emphasis on Cost 20:20:60": 61.35547295412263,
    "Emphasis on Speed 20:60:20": 74.7620726966441,
    "Intelligence only": 63.620757840247784
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "TypeSafe AI",
   "axes": {
    "calibration": 90.08195706553009,
    "cost": 54.815314343766296,
    "intelligence": 59.942981848430136,
    "speed": 90.98627348971067
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.040476900149031346,
      "ece_score": 91.90461997019374,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 91.90461997019374,
      "score_before_time_limit": 91.90461997019374,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.24128686327077753,
      "brier": 0.08160294906166222,
      "ece": 0.06619302949061674,
      "n": 373,
      "score": 86.76139410187666,
      "score_before_time_limit": 86.76139410187666,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9731182795698925,
      "interval90_width": 0.2782258064516129,
      "n": 372,
      "nrps": 0.06044260987193544,
      "score": 91.57985712451988,
      "score_before_time_limit": 91.57985712451988,
      "time_limit_failed_share": 0,
      "top_ece": 0.053980123818833534
     }
    },
    "score": 90.08195706553009
   },
   "capability": 75.01246945698011,
   "composite_ci95": {
    "A": [
     67.73670723361865,
     71.21043212518939
    ],
    "B": [
     64.61448469474541,
     69.61304774005897
    ],
    "C": [
     55.864974598132086,
     71.21043212518939
    ]
   },
   "cost": {
    "basis": "TypeSafe AI public list price, USD 0.042/M input, output free: the \"Current models\" table of TypeSafe's models page (docs.typesafe.ai/models, fetched 16 Oct 2026 22:37:44Z, saved copy sha256 936da179...) lists jev-1.14.0 at \"$42 / $0.042\" per Btok / per Mtok, the same price as jev-1.13.0, and says \"You pay per input token. Output tokens are free.\"; Jev 1.14 is \"available from October 14, 2026\". Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap, d1-ultra, Sage 1.4, Nimble, kettei and Celeris-1 Decision), through a scoped re-freeze of the frozen pricing module that adds this one key and nothing else, with a zero-difference proof for every other registry entry; jev-1.13.0 is priced through the same launch-list set at the same tariff. On the same page the alias jev-latest (\"the most recent stable, official release\") still points to jev-1.13.0, and jev-preview (\"the most recent release, official or not\") points to jev-1.14.0; the versioned id is listed with its price, accepted by the API and was the served model on every row. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (1,134,053 input tokens over the 1,485 answered items; the 71,566 reported output tokens are free under the tariff), divided by the 1,485 items of the common cost basis the scorer uses; the 15 requests refused with HTTP 400 report no usage and count as zero tokens and as wrong answers (TypeSafe's docs do not say whether refused requests are billed). USD 0.032074 per 1,000 decisions on the common basis (USD 0.031753 over all 1,500 items), 0.497x the official cost cap. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.032074 per 1,000 decisions (0.497x the official cost cap USD 0.064595), against USD 0.031753 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.031753484
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.032074226262626265
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jev 1.14.0 (TypeSafe AI)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted TypeSafe AI API (api.typesafe.ai), model `jev-1.14.0` (served model id `jev-1.14.0` on all 1,500 responses); latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of `v1.6-regular-20261012` through the unchanged `run_api_full_v161.py` and the unchanged `jevbench.adapters.typesafe`; the full-set exposure gate logged all 1,200 sealed items to provider `typesafe` before the first request (under the pre-registration). Native probabilities on all three question types. 15 of 1,500 requests returned HTTP 400 max_tokens_exceeded (the 15 long items of this draw; per TypeSafe's API reference the request is longer than the model's context length, 64k per request by default); the runner never re-sends a failed item, so they are scored as wrong in the full 1,500 denominator. TypeSafe documents an opt-in long-context mode (request header TypeSafe-Danger-Zone-Long-Context: true, up to 262k tokens at the same per-token price); this run used the default 64k context, as every hosted row of this draw used its default, so the 15 longest items were refused. 12 answered rows return probabilities that sum to 0.99 (the API rounds to two decimals); the unchanged scorer renormalises distributions within its 2e-2 tolerance, as it does for the published jev-1.13.0 raw, which has the same two-decimal property. Hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 66.09089256935101,
    "I_sealed": 53.79507112750927,
    "base": 59.942981848430136,
    "excess": 2.4562946714860345,
    "gap": 12.295821441841746,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 79.38533059643669,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 73.57659641396664,
       "judge": 84.17930125247199,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 52.73567977915805,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 19.999999999999996,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 66.1516673324583,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 69,
       "hard": 71.16105499438834,
       "judge": 74.77229926382469,
       "standard": 41.77777777777777
      }
     },
     "sealed|choice": {
      "cc": 70.9152814841789,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 65.13241586677461,
       "judge": 64.74045053868758,
       "standard": 81.96582999367222
      }
     },
     "sealed|noul": {
      "cc": 37.13412680988595,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 29.0909090909091,
       "judge": 32.30769230769231,
       "standard": 46.26865671641791
      }
     },
     "sealed|score": {
      "cc": 53.33580508846296,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 79.47368421052632,
       "hard": 50.35293782899417,
       "judge": 53.192708333333314,
       "standard": 46.44724517906332
      }
     }
    }
   },
   "jevbench_score": 70.1541854578216,
   "key": "jev-1.14.0",
   "last_measured_on": "2026-10-16",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.27",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.typesafe.ai/v1/systemone#jev-1.14.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9540636042402827,
    "decisive": 283,
    "decisive_rate": 0.7587131367292225,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 16,
    "axes": {
     "calibration": 90.08195706553009,
     "cost": 54.815314343766296,
     "intelligence": 59.942981848430136,
     "speed": 90.96457246082724
    },
    "board_release": "v1.6.43",
    "capability": 75.01246945698011,
    "capability_api_board_rank": 10,
    "capability_open_board_rank": null,
    "capability_rank": 23,
    "composite_ci95": {
     "A": [
      67.7336997653583,
      71.20710829258444
     ],
     "B": [
      64.61229538497312,
      69.61050660584631
     ],
     "C": [
      55.86249334963948,
      71.20710829258444
     ]
    },
    "jevbench_score": 70.1509594948736,
    "measured_in": "v1.6.27",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 26,
    "ranks": {
     "A": 26,
     "B": 31,
     "C": 24,
     "capability": 43
    },
    "scores": {
     "A": 70.1509594948736,
     "B": 67.84038658562716,
     "C": 70.01769357840206
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.2238923824916128,
     "p50_s_raw": 0.2238923824916128,
     "p95_s_adjusted": 0.3576872305740834,
     "p95_s_raw": 0.3576872305740834
    },
    "views": {
     "Balanced 33:33:33": 65.33259265119166,
     "Emphasis on Accuracy 60:20:20": 63.06448377477604,
     "Emphasis on Cost 20:20:60": 60.67589899590322,
     "Emphasis on Speed 20:60:20": 73.63177120780584,
     "Intelligence only": 59.94298184843013
    }
   },
   "rank": 75,
   "ranked": true,
   "ranks": {
    "A": 75,
    "B": 71,
    "C": 60,
    "capability": 62
   },
   "repo": "https://docs.typesafe.ai/models",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.1541854578216,
    "B": 67.8428001241696,
    "C": 70.02091341298186
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22666933800792322,
    "p50_s_raw": 0.22666933800792322,
    "p95_s_adjusted": 0.35154414957505653,
    "p95_s_raw": 0.35154414957505653
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 48.444444444444436,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": -37.999999999999986
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 48.444444444444436
      },
      "calibration_gen": {
       "accuracy": 0.8415841584158416,
       "cc": 75.80772107786954,
       "cc_by_type": {
        "choice": 90.45599151643691,
        "noul": 76,
        "score": 60.96717171717172
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 75.80772107786954
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 83.08601440715262,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 85.18518518518519,
        "score": 70.57692307692307
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 83.08601440715262
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.62962962962963,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 90.88888888888889
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.62962962962963
      },
      "jaggedness": {
       "accuracy": 0.6388888888888888,
       "cc": 41.63374516451571,
       "cc_by_type": {
        "choice": 58.132820019249294,
        "noul": 13.513513513513509,
        "score": 53.254901960784316
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 41.63374516451571
      },
      "llm_router": {
       "accuracy": 0.805111821086262,
       "cc": 69.88977768858938,
       "cc_by_type": {
        "choice": 69.8864136655807,
        "noul": 85.56701030927836,
        "score": 54.2159090909091
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 69.88977768858938
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4444444444444444,
       "cc": 4.824284408476845,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -50,
        "score": 26.328523328523325
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 4.824284408476845
      },
      "multilingual": {
       "accuracy": 0.6727272727272727,
       "cc": 40.78295167895502,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": -5.555555555555558,
        "score": 58.22720159339878
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 40.78295167895502
      },
      "negation_pair_gen": {
       "accuracy": 0.872093023255814,
       "cc": 71.22659932659933,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 53.679797979797975
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 71.22659932659933
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 81.58273658273659,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100,
        "score": 65.76923076923077
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 81.58273658273659
      },
      "usecase": {
       "accuracy": 0.6772727272727272,
       "cc": 45.09244048326402,
       "cc_by_type": {
        "choice": 71.81372549019609,
        "noul": 8.333333333333325,
        "score": 55.130262626262635
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 45.09244048326402
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -3.8035668035668095,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": -50,
        "score": -23.315462315462334
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.46153846153846156,
       "cc": 14.130670762928817,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": 17.19999999999998
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 14.130670762928817
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 75.16583301269719,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 74.3913043478261
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 75.16583301269719
      },
      "el": {
       "accuracy": 0.6,
       "cc": 44.21570453134698,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -33.333333333333336,
        "score": 99.49999999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.21570453134698
      },
      "en": {
       "accuracy": 0.7767158992180713,
       "cc": 63.373471305868186,
       "cc_by_type": {
        "choice": 75.17726168828789,
        "noul": 56.11510791366907,
        "score": 58.82804431564761
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 63.373471305868186
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 72.85500431820361,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 28.35294117647056
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 72.85500431820361
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 31.91985428051002,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 94.66666666666667
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 31.91985428051002
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 74.20949432404541,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 86.15789473684211
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 74.20949432404541
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 38.826815642458094,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 49.99999999999998
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 38.826815642458094
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 51.4564450877779,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": -17.57142857142857
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 51.4564450877779
      },
      "ja": {
       "accuracy": 0.4,
       "cc": 0.3594740093548647,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -19.999999999999996,
        "score": -21.733333333333338
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.3594740093548647
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 1.9232804232804217,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 50.21428571428571
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 1.9232804232804217
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.44444444444444,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.44444444444444
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -6.1826839826839874,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 33.399999999999984
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 42.02553867862059,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 61.31148696264975
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 42.02553867862059
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 73.3822938645681,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 51.96774193548389
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 73.3822938645681
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 22.968085106382983,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 65.5
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 22.968085106382983
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 27.670289855072465,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0,
        "score": 4.749999999999998
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 27.670289855072465
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 77.66129032258064,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 97.49999999999999
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 77.66129032258064
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 31.88888888888889,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 29.000000000000004
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 31.88888888888889
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7946666666666666,
       "cc": 73.47825254980003,
       "cc_by_type": {
        "choice": 73.47825254980003
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 73.47825254980003
      },
      "noul": {
       "accuracy": 0.72,
       "cc": 43.99999999999999,
       "cc_by_type": {
        "noul": 43.99999999999999
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 43.99999999999999
      },
      "score": {
       "accuracy": 0.6933333333333334,
       "cc": 56.740655654013075,
       "cc_by_type": {
        "score": 56.740655654013075
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 56.740655654013075
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": 6.09803921568628,
       "cc_by_type": {
        "choice": 68,
        "noul": -75,
        "score": 25.294117647058833
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 6.09803921568628
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 31.665734426604,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 55.55555555555556,
        "score": -4.6577311577311376
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 31.665734426604
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 38.40650406504064,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 13.999999999999979
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 38.40650406504064
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -28.41544834307992,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -3.3749999999999947
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 30.187801932367147,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 67.37499999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 30.187801932367147
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": 13.061844863731627,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 0,
        "score": -10.500000000000087
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 13.061844863731627
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 49.6704014939309,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 54.33333333333332
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 49.6704014939309
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 29.958895116901484,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": -50,
        "score": 60.78260869565217
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 29.958895116901484
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 18.722222222222225,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 78.66666666666667
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.722222222222225
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 38.48014405294919,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 0,
        "score": 41.13636363636364
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 38.48014405294919
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 43.90517578889672,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 54.45454545454547
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 43.90517578889672
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -8.535842293906812,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": 4.499999999999993
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 71.50523756079312,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 54.51571268237934
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 71.50523756079312
      },
      "model_routing": {
       "accuracy": 0.8294930875576036,
       "cc": 70.61857413419914,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 90.90909090909092,
        "score": 40.81676136363635
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 70.61857413419914
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 8.93466561288623,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 61.3529411764706
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 8.93466561288623
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 31.887853248577482,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 62.423076923076934
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 31.887853248577482
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 9.766154737757171,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50,
        "score": 38.41176470588235
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 9.766154737757171
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 9.087439613526568,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -66.66666666666667,
        "score": 51.89999999999999
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 9.087439613526568
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 83.06140350877193,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 99.18421052631578
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 83.06140350877193
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.10088425105317,
      90.38371712636082
     ],
     "I": [
      54.572148574825896,
      63.98122322055633
     ],
     "capability": [
      70.67669877479854,
      76.35848785536061
     ],
     "composite": {
      "A": [
       67.73670723361865,
       71.21043212518939
      ],
      "B": [
       64.61448469474541,
       69.61304774005897
      ],
      "C": [
       55.864974598132086,
       71.21043212518939
      ]
     },
     "seed": 36
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.08195706553009,
     "capability": 75.01246945698011,
     "intelligence": 59.942981848430136
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7946666666666666,
      "calibration": 91.90461997019374,
      "cc": 73.47825254980003,
      "cc_by_type": {
       "choice": 73.47825254980003
      },
      "cc_tiered": 75.1503060403078,
      "cc_tiered_open": 79.38533059643669,
      "cc_tiered_sealed": 70.9152814841789,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.47825254980003
     },
     "noul": {
      "accuracy": 0.72,
      "calibration": 86.76139410187666,
      "cc": 43.99999999999999,
      "cc_by_type": {
       "noul": 43.99999999999999
      },
      "cc_tiered": 44.934903294522,
      "cc_tiered_open": 52.73567977915805,
      "cc_tiered_sealed": 37.13412680988595,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 43.99999999999999
     },
     "score": {
      "accuracy": 0.6933333333333334,
      "calibration": 91.57985712451988,
      "cc": 56.740655654013075,
      "cc_by_type": {
       "score": 56.740655654013075
      },
      "cc_tiered": 59.74373621046063,
      "cc_tiered_open": 66.1516673324583,
      "cc_tiered_sealed": 53.33580508846296,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 56.740655654013075
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.33632339558339,
    "Emphasis on Accuracy 60:20:20": 63.066569447746595,
    "Emphasis on Cost 20:20:60": 60.67782966754747,
    "Emphasis on Speed 20:60:20": 73.64030150613897,
    "Intelligence only": 59.94298184843013
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "AutoTrust AI Lab",
   "axes": {
    "calibration": 90.89967561034447,
    "cost": 58.5653810087091,
    "intelligence": 56.828664559360675,
    "speed": 86.4682823414372
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.044592591608484194,
      "ece_score": 91.08148167830316,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 91.08148167830316,
      "score_before_time_limit": 91.08148167830316,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.21293800539083563,
      "brier": 0.0834318516755718,
      "ece": 0.0453231523993617,
      "n": 371,
      "score": 90.93536952012767,
      "score_before_time_limit": 90.93536952012767,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9539295392953929,
      "interval90_width": 0.2831978319783197,
      "n": 369,
      "nrps": 0.07252867726920081,
      "score": 90.68217563260262,
      "score_before_time_limit": 90.68217563260262,
      "time_limit_failed_share": 0,
      "top_ece": 0.05691390503937325
     }
    },
    "score": 90.89967561034447
   },
   "capability": 73.86417008485257,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 689,734 input + 1,477 output tokens over 1,477 decisions = 467.983073798 tokens/decision. GPU-time cost USD 0.0240521534746298/1,000; old list-price cost USD 0.199132890995261/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0240521534746298/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.19607951999999995
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.024052153474629835
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JEV-27B (AutoTrust, Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 57.748245429372346,
    "I_sealed": 55.909083689349,
    "base": 56.828664559360675,
    "excess": -0.7342259024149009,
    "gap": 1.8391617400233429,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 77.03865189535385,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 75.46398238439761,
       "judge": 76.26895187870797,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 45.291235334713605,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 10.000000000000009,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 50.91484905804961,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.85808404482206,
       "hard": 54.85367186261314,
       "judge": 58.70222464767498,
       "standard": 25.384522571098287
      }
     },
     "sealed|choice": {
      "cc": 72.15078699038692,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 77.38526346907102,
       "hard": 66.4027149321267,
       "judge": 72.06703910614526,
       "standard": 81.15531469392782
      }
     },
     "sealed|noul": {
      "cc": 42.779753431164394,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72,
       "hard": 24.271844660194162,
       "judge": 42.10526315789473,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 52.79671064649569,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 58.73482464128581,
       "hard": 48.652428202150055,
       "judge": 63.82621512662949,
       "standard": 41.571961817591195
      }
     }
    }
   },
   "jevbench_score": 69.88393575514445,
   "key": "jev-27b-autotrust",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "autotrust/JEV-27B@51740a8891c2a8baefd969237fd44187b3e3a115",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9452054794520548,
    "decisive_rate": 0.7870619946091644,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.89967561034447,
     "cost": 31.02571003473804,
     "intelligence": 56.828664559360675,
     "speed": 86.89503266505372
    },
    "board_release": "v1.6.43",
    "capability": 73.86417008485257,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.660744990021566,
      21.639609973811442
     ],
     "B": [
      20.50360595860232,
      22.019717921334916
     ],
     "C": [
      15.257925180346783,
      21.639609973811442
     ]
    },
    "jevbench_score": 21.291230595611577,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 88,
    "rank": 122,
    "ranks": {
     "A": 122,
     "B": 121,
     "C": 117,
     "capability": 49
    },
    "scores": {
     "A": 21.291230595611577,
     "B": 21.406663290369472,
     "C": 19.09999105447478
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3604971420194488,
     "p50_s_raw": 0.1052485710097244,
     "p95_s_adjusted": 0.5670154692983488,
     "p95_s_raw": 0.20850773464917438
    },
    "views": {
     "Balanced 33:33:33": 18.83248652432942,
     "Emphasis on Accuracy 60:20:20": 19.94400418862255,
     "Emphasis on Cost 20:20:60": 15.303706986099348,
     "Emphasis on Speed 20:60:20": 22.82314538634501,
     "Intelligence only": 21.88118814236141
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 79,
   "ranked": true,
   "ranks": {
    "A": 79,
    "B": 82,
    "C": 79,
    "capability": 67
   },
   "repo": "https://huggingface.co/autotrust/JEV-27B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.88393575514445,
    "B": 66.81409036571809,
    "C": 62.69165804112172
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.37615348198451104,
    "p50_s_raw": 0.11307674099225551,
    "p95_s_adjusted": 0.5995241465250728,
    "p95_s_raw": 0.22476207326253642
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.12231981093008,
      91.47472229481626
     ],
     "I": [
      51.54879559651137,
      61.562027595056925
     ],
     "capability": [
      69.671164690791,
      75.76898842923057
     ],
     "composite": null,
     "seed": 51
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.89967561034447,
     "capability": 73.86417008485257,
     "intelligence": 56.828664559360675
    },
    "per_type": {
     "choice": {
      "accuracy": 0.796,
      "calibration": 91.08148167830316,
      "cc": 73.91131362314447,
      "cc_by_type": {
       "choice": 73.91131362314447
      },
      "cc_tiered": 74.59471944287039,
      "cc_tiered_open": 77.03865189535385,
      "cc_tiered_sealed": 72.15078699038692,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.91131362314447
     },
     "noul": {
      "accuracy": 0.736,
      "calibration": 90.93536952012767,
      "cc": 47.199999999999996,
      "cc_by_type": {
       "noul": 47.199999999999996
      },
      "cc_tiered": 44.035494382939,
      "cc_tiered_open": 45.291235334713605,
      "cc_tiered_sealed": 42.779753431164394,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 47.199999999999996
     },
     "score": {
      "accuracy": 0.6506666666666666,
      "calibration": 90.68217563260262,
      "cc": 51.866475511741164,
      "cc_by_type": {
       "score": 51.866475511741164
      },
      "cc_tiered": 51.85577985227265,
      "cc_tiered_open": 50.91484905804961,
      "cc_tiered_sealed": 52.79671064649569,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 51.866475511741164
     }
    },
    "raw_on_A": {
     "calibration": 88.71172832565286,
     "capability": 73.81510020130236,
     "intelligence": 58.91847207695186
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.88363348754837,
    "Emphasis on Accuracy 60:20:20": 61.402334729605386,
    "Emphasis on Cost 20:20:60": 62.19950724069505,
    "Emphasis on Speed 20:60:20": 72.08091404704541,
    "Intelligence only": 56.828664559360675
   }
  },
  {
   "adapter_id": "images/jev-local/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "us (GitHub)",
   "axes": {
    "calibration": 73.06634984647151,
    "cost": 78.54608301342061,
    "intelligence": 41.85041350559534,
    "speed": 72.89461160481758
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.16559413680781765,
      "ece_score": 66.88117263843647,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 66.88117263843647,
      "score_before_time_limit": 66.88117263843647,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.14555256064690025,
      "brier": 0.16930220654986547,
      "ece": 0.14746549865229122,
      "n": 371,
      "score": 70.50690026954176,
      "score_before_time_limit": 70.50690026954176,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8644986449864499,
      "interval90_width": 0.325880758807588,
      "n": 369,
      "nrps": 0.10296637523035226,
      "score": 81.81097663143632,
      "score_before_time_limit": 81.81097663143632,
      "time_limit_failed_share": 0,
      "top_ece": 0.1304070460704606
     }
    },
    "score": 73.06634984647151
   },
   "capability": 57.458381676033426,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 379,568 input + 1,624 output tokens over 1,624 decisions = 234.724137931 tokens/decision. GPU-time cost USD 0.00518956034332633/1,000; old list-price cost USD 0.0235224137931034/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00518956034332633/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0051895603433263346
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jev-local (Qwen3.5-9B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 43.39498650106927,
    "I_sealed": 40.30584051012142,
    "base": 41.85041350559534,
    "excess": 0.5157583485096047,
    "gap": 3.0891459909478485,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 56.18661240978703,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.2972972972973,
       "hard": 49.040578798364265,
       "judge": 52.537903757415954,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 34.189786059351285,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": 30.000000000000004,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 39.8085610340695,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 40.41969696969697,
       "hard": 51.80083333333333,
       "judge": 22.97271186440677,
       "standard": 40.77222222222221
      }
     },
     "sealed|choice": {
      "cc": 47.63585353447873,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 58.7613627965413,
       "hard": 39.42307692307691,
       "judge": 48.78957169459964,
       "standard": 56.76807488606972
      }
     },
     "sealed|noul": {
      "cc": 35.679356877082625,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": 14.563106796116498,
       "judge": 42.10526315789473,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 37.602311118802916,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 64.23531914893618,
       "hard": 29.158455882352953,
       "judge": 39.61335329341317,
       "standard": 38.15695431472081
      }
     }
    }
   },
   "jevbench_score": 43.76548052831858,
   "key": "jev-local",
   "last_measured_on": "2026-10-02",
   "licence": "no licence stated in the repository (public code); Apache-2.0 base weights",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8233438485804416,
    "decisive_rate": 0.8544474393530997,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.06634984647151,
     "cost": 58.85554343293177,
     "intelligence": 41.85041350559534,
     "speed": 75.55955847735143
    },
    "board_release": "v1.6.43",
    "capability": 57.458381676033426,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 67,
    "capability_rank": 92,
    "composite_ci95": {
     "A": [
      29.136540211681222,
      52.90884989606441
     ],
     "B": [
      26.3171992505717,
      49.89250384145016
     ],
     "C": [
      20.233708480334183,
      36.74225687226694
     ]
    },
    "jevbench_score": 41.328564334669615,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 52,
    "rank": 79,
    "ranks": {
     "A": 79,
     "B": 81,
     "C": 83,
     "capability": 146
    },
    "scores": {
     "A": 41.328564334669615,
     "B": 38.19938992160457,
     "C": 28.700391899076113
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.8806604199111462,
     "p50_s_raw": 0.3653302099555731,
     "p95_s_adjusted": 3.156717182928696,
     "p95_s_raw": 1.5033585914643481
    },
    "views": {
     "Balanced 33:33:33": 38.834988968034565,
     "Emphasis on Accuracy 60:20:20": 34.37288631458028,
     "Emphasis on Cost 20:20:60": 39.75999364263786,
     "Emphasis on Speed 20:60:20": 43.46632094435822,
     "Intelligence only": 29.31968172619126
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 136,
   "ranked": true,
   "ranks": {
    "A": 136,
    "B": 137,
    "C": 142,
    "capability": 178
   },
   "repo": "https://github.com/us/jev-local",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 43.76548052831858,
    "B": 39.83968067331161,
    "C": 30.39269481133234
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.0924282498657703,
    "p50_s_raw": 0.47121412493288517,
    "p95_s_adjusted": 4.700520133133978,
    "p95_s_raw": 2.2752600665669886
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-9B, frozen, per-option mean log-probability",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.57161963107492,
      76.81314888025398
     ],
     "I": [
      36.13732889468463,
      46.54627874317067
     ],
     "capability": [
      53.19292305173127,
      60.44357521274037
     ],
     "composite": null,
     "seed": 45
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.06634984647151,
     "capability": 57.458381676033426,
     "intelligence": 41.85041350559534
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6026666666666667,
      "calibration": 66.88117263843647,
      "cc": 49.186741566647385,
      "cc_by_type": {
       "choice": 49.186741566647385
      },
      "cc_tiered": 51.91123297213288,
      "cc_tiered_open": 56.18661240978703,
      "cc_tiered_sealed": 47.63585353447873,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.186741566647385
     },
     "noul": {
      "accuracy": 0.696,
      "calibration": 70.50690026954176,
      "cc": 39.19999999999999,
      "cc_by_type": {
       "noul": 39.19999999999999
      },
      "cc_tiered": 34.93457146821696,
      "cc_tiered_open": 34.189786059351285,
      "cc_tiered_sealed": 35.679356877082625,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.19999999999999
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 81.81097663143632,
      "cc": 39.00943859649125,
      "cc_by_type": {
       "score": 39.00943859649125
      },
      "cc_tiered": 38.70543607643621,
      "cc_tiered_open": 39.8085610340695,
      "cc_tiered_sealed": 37.602311118802916,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 39.00943859649125
     }
    },
    "raw_on_A": {
     "calibration": 73.6085835138451,
     "capability": 59.88003789504907,
     "intelligence": 46.151492276253045
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 41.747380261651486,
    "Emphasis on Accuracy 60:20:20": 35.69532613064949,
    "Emphasis on Cost 20:20:60": 46.20819921292856,
    "Emphasis on Speed 20:60:20": 45.035422445111884,
    "Intelligence only": 29.31968172619126
   }
  },
  {
   "adapter_id": "images/jev-omni/spec.json:omni_v15.JevOmni",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "akhilaaa3",
   "axes": {
    "calibration": 87.04450672835743,
    "cost": 66.45108408007525,
    "intelligence": 55.530225828826175,
    "speed": 84.73829853475769
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10691343524556984,
      "ece_score": 78.61731295088603,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 78.61731295088603,
      "score_before_time_limit": 78.61731295088603,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.25876010781671155,
      "brier": 0.09426879548573859,
      "ece": 0.04284927173453476,
      "n": 371,
      "score": 91.43014565309304,
      "score_before_time_limit": 91.43014565309304,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.94579945799458,
      "interval90_width": 0.32836495031616986,
      "n": 369,
      "nrps": 0.06732883113404022,
      "score": 91.08606158109325,
      "score_before_time_limit": 91.08606158109325,
      "time_limit_failed_share": 0,
      "top_ece": 0.0554749686220474
     }
    },
    "score": 87.04450672835743
   },
   "capability": 71.2873662785918,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 947,498 input + 0 output tokens over 1,624 decisions = 583.434729064 tokens/decision. GPU-time cost USD 0.0131309817378658/1,000; old list-price cost USD 0.029171736453202/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0131309817378658/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013130981737865803
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jev-Omni (akhilaaa3, Gemma-4-12B merged)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 59.14991720973798,
    "I_sealed": 51.91053444791438,
    "base": 55.530225828826175,
    "excess": 4.6659951193853555,
    "gap": 7.239382761823599,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.02831016136747,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 73.57659641396666,
       "judge": 68.35860250494396,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 45.118702553485164,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 65.21739130434783,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 57.30273891436131,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.591582585888155,
       "hard": 52.24401347123031,
       "judge": 64.7750828511346,
       "standard": 55.56725205969997
      }
     },
     "sealed|choice": {
      "cc": 68.8767390165768,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 62.33031674208144,
       "judge": 74.86033519553072,
       "standard": 70.07020569035596
      }
     },
     "sealed|noul": {
      "cc": 32.45047104291564,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48,
       "hard": 12.621359223300965,
       "judge": 36.8421052631579,
       "standard": 57.746478873239425
      }
     },
     "sealed|score": {
      "cc": 54.404393284250695,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 74.52093494591816,
       "hard": 48.39164613508641,
       "judge": 58.117448525933035,
       "standard": 50.80203388922203
      }
     }
    }
   },
   "jevbench_score": 70.98959059698258,
   "key": "jev-omni",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0, following Gemma 4; dataset rights stated separately by the author",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9381818181818182,
    "decisive_rate": 0.7412398921832885,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.04450672835743,
     "cost": 56.05113155939739,
     "intelligence": 55.530225828826175,
     "speed": 85.43173201875288
    },
    "board_release": "v1.6.43",
    "capability": 71.2873662785918,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 21,
    "capability_rank": 38,
    "composite_ci95": {
     "A": [
      65.25170208143348,
      68.96455653744843
     ],
     "B": [
      61.61186910919615,
      66.73364708003527
     ],
     "C": [
      45.97947208044005,
      67.32404729169104
     ]
    },
    "jevbench_score": 67.74758858028281,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 20,
    "rank": 40,
    "ranks": {
     "A": 40,
     "B": 42,
     "C": 42,
     "capability": 68
    },
    "scores": {
     "A": 67.74758858028281,
     "B": 64.89216659285307,
     "C": 58.02968592150493
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.45938353668898346,
     "p50_s_raw": 0.15469176834449172,
     "p95_s_adjusted": 0.6232343347743153,
     "p95_s_raw": 0.23661716738715766
    },
    "views": {
     "Balanced 33:33:33": 63.085756293696,
     "Emphasis on Accuracy 60:20:20": 59.82955470935818,
     "Emphasis on Cost 20:20:60": 60.07014592127774,
     "Emphasis on Speed 20:60:20": 70.45744510674369,
     "Intelligence only": 55.53022582882618
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 67,
   "ranked": true,
   "ranks": {
    "A": 67,
    "B": 77,
    "C": 83,
    "capability": 86
   },
   "repo": "https://huggingface.co/akhilaaa3/Jev-Omni",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.98959059698258,
    "B": 67.24542585851624,
    "C": 60.80664614589772
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4731816291809082,
    "p50_s_raw": 0.1615908145904541,
    "p95_s_adjusted": 0.7098102578893304,
    "p95_s_raw": 0.2799051289446652
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it fine-tuned and merged, with a trained 256-way decision head",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.22517864881327,
      88.34500746248735
     ],
     "I": [
      50.30228764136711,
      59.27245651053757
     ],
     "capability": [
      67.20628773376218,
      73.03534010513394
     ],
     "composite": null,
     "seed": 46
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.04450672835743,
     "capability": 71.2873662785918,
     "intelligence": 55.530225828826175
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7693333333333333,
      "calibration": 78.61731295088603,
      "cc": 70.5010278222483,
      "cc_by_type": {
       "choice": 70.5010278222483
      },
      "cc_tiered": 71.95252458897214,
      "cc_tiered_open": 75.02831016136747,
      "cc_tiered_sealed": 68.8767390165768,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.5010278222483
     },
     "noul": {
      "accuracy": 0.688,
      "calibration": 91.43014565309304,
      "cc": 37.59999999999999,
      "cc_by_type": {
       "noul": 37.59999999999999
      },
      "cc_tiered": 38.7845867982004,
      "cc_tiered_open": 45.118702553485164,
      "cc_tiered_sealed": 32.45047104291564,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 37.59999999999999
     },
     "score": {
      "accuracy": 0.6906666666666667,
      "calibration": 91.08606158109325,
      "cc": 55.14278895136254,
      "cc_by_type": {
       "score": 55.14278895136254
      },
      "cc_tiered": 55.85356609930601,
      "cc_tiered_open": 57.30273891436131,
      "cc_tiered_sealed": 54.404393284250695,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 55.14278895136254
     }
    },
    "raw_on_A": {
     "calibration": 85.06452164862384,
     "capability": 71.04516964846138,
     "intelligence": 57.025817648298926
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.877832046081,
    "Emphasis on Accuracy 60:20:20": 61.824310842967385,
    "Emphasis on Cost 20:20:60": 66.70647680821602,
    "Emphasis on Speed 20:60:20": 73.0353573955765,
    "Intelligence only": 55.53022582882618
   }
  },
  {
   "adapter_id": "images/jev-qwen3.5-9b-base-nvfp4/spec.json:jev_choice_http.Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "WilfLin",
   "axes": {
    "calibration": 85.03288206087805,
    "cost": 47.592235403520306,
    "intelligence": 29.251131763478327,
    "speed": 93.41131234211913
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05599448584772476,
      "ece_score": 88.80110283045505,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.80110283045505,
      "score_before_time_limit": 88.80110283045505,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.47439353099730464,
      "brier": 0.15464738864390087,
      "ece": 0.10676843098382341,
      "n": 371,
      "score": 78.64631380323533,
      "score_before_time_limit": 78.64631380323533,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.975609756097561,
      "interval90_width": 0.6402439024390243,
      "n": 369,
      "nrps": 0.10255360461757664,
      "score": 87.65122954894377,
      "score_before_time_limit": 87.65122954894377,
      "time_limit_failed_share": 0,
      "top_ece": 0.07221090220177398
     }
    },
    "score": 85.03288206087805
   },
   "capability": 57.14200691217819,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     10.55100535128086,
     22.568544734606675
    ],
    "B": [
     8.922738700914902,
     20.044253440413932
    ],
    "C": [
     7.327087049500598,
     15.67260051014353
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (proxy tokens, I-2): exact base-model market reference",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05583743842364531
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JEV Qwen3.5-9B Base NVFP4",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 30.95041263176777,
    "I_sealed": 27.551850895188885,
    "base": 29.251131763478327,
    "excess": 0.8251740941406402,
    "gap": 3.398561736578884,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 55.43165802161466,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.2972972972973,
       "hard": 47.153192827933324,
       "judge": 52.537903757415954,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 3.6694271911663243,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -21.739130434782606,
       "judge": 10.000000000000009,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 33.750152682522334,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.19655618503351,
       "hard": 33.2215341463409,
       "judge": 26.48323152009081,
       "standard": 33.48456974727688
      }
     },
     "sealed|choice": {
      "cc": 47.891397457772825,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 66.74303451333975,
       "hard": 41.96832579185519,
       "judge": 45.065176908752335,
       "standard": 54.55105308535534
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 24,
       "hard": -26.21359223300971,
       "judge": -18.42105263157895,
       "standard": -9.859154929577462
      }
     },
     "sealed|score": {
      "cc": 34.764155227793836,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 59.111326959327606,
       "hard": 22.60553224460797,
       "judge": 38.31832027118539,
       "standard": 41.57656776331136
      }
     }
    }
   },
   "jevbench_score": 15.970464603997094,
   "key": "jev-qwen3.5-9b-base-nvfp4",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 entrant and upstream checkpoint",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8923076923076924,
    "decisive_rate": 0.5256064690026954,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.03288206087805,
     "cost": 47.592235403520306,
     "intelligence": 29.251131763478327,
     "speed": 93.62779491136789
    },
    "board_release": "v1.6.43",
    "capability": 57.14200691217819,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 69,
    "capability_rank": 95,
    "composite_ci95": {
     "A": [
      10.554100896173676,
      22.576213316295057
     ],
     "B": [
      8.924509602976766,
      20.049092209565615
     ],
     "C": [
      7.329236733453941,
      15.67792591409379
     ]
    },
    "jevbench_score": 15.97555620862577,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 116,
    "rank": 154,
    "ranks": {
     "A": 154,
     "B": 156,
     "C": 154,
     "capability": 149
    },
    "scores": {
     "A": 15.97555620862577,
     "B": 13.864517363543307,
     "C": 11.094136255990119
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19534559999592602,
     "p50_s_raw": 0.02267279999796301,
     "p95_s_adjusted": 0.22203266686410641,
     "p95_s_raw": 0.03601633343205321
    },
    "views": {
     "Balanced 33:33:33": 14.120527794466737,
     "Emphasis on Accuracy 60:20:20": 11.548484256330752,
     "Emphasis on Cost 20:20:60": 14.368613309854894,
     "Emphasis on Speed 20:60:20": 17.771756183840008,
     "Intelligence only": 9.070269959573535
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 203,
   "ranked": true,
   "ranks": {
    "A": 203,
    "B": 199,
    "C": 203,
    "capability": 181
   },
   "repo": "https://huggingface.co/WIlfLin/JEV-Qwen3.5-9B-Base-NVFP4",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 15.970464603997094,
    "B": 13.861449157550995,
    "C": 11.090600419442426
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19610175802372395,
    "p50_s_raw": 0.02305087901186198,
    "p95_s_adjusted": 0.23248090025736018,
    "p95_s_raw": 0.04124045012868009
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-9B Base NVFP4 with compact BF16 decision head",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.95850301005234,
      86.17596383374455
     ],
     "I": [
      24.784047457667274,
      33.682401804173935
     ],
     "capability": [
      52.832944781851424,
      58.83201835103813
     ],
     "composite": {
      "A": [
       10.55100535128086,
       22.568544734606675
      ],
      "B": [
       8.922738700914902,
       20.044253440413932
      ],
      "C": [
       7.327087049500598,
       15.67260051014353
      ]
     },
     "seed": 47
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.03288206087805,
     "capability": 57.14200691217819,
     "intelligence": 29.251131763478327
    },
    "per_type": {
     "choice": {
      "accuracy": 0.608,
      "calibration": 88.80110283045505,
      "cc": 49.86879872682661,
      "cc_by_type": {
       "choice": 49.86879872682661
      },
      "cc_tiered": 51.66152773969374,
      "cc_tiered_open": 55.43165802161466,
      "cc_tiered_sealed": 47.891397457772825,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.86879872682661
     },
     "noul": {
      "accuracy": 0.464,
      "calibration": 78.64631380323533,
      "cc": -7.199999999999996,
      "cc_by_type": {
       "noul": -7.199999999999996
      },
      "cc_tiered": 1.8347135955831622,
      "cc_tiered_open": 3.6694271911663243,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.5413333333333333,
      "calibration": 87.65122954894377,
      "cc": 35.19517866359504,
      "cc_by_type": {
       "score": 35.19517866359504
      },
      "cc_tiered": 34.25715395515809,
      "cc_tiered_open": 33.750152682522334,
      "cc_tiered_sealed": 34.764155227793836,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.19517866359504
     }
    },
    "raw_on_A": {
     "calibration": 80.94123658739136,
     "capability": 56.72695595498426,
     "intelligence": 32.51267532257717
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.115224343155328,
    "Emphasis on Accuracy 60:20:20": 11.546355426093731,
    "Emphasis on Cost 20:20:60": 14.365317962330064,
    "Emphasis on Speed 20:60:20": 17.75664203960864,
    "Intelligence only": 9.070269959573535
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 579,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "einptein",
   "axes": {
    "calibration": 47.44,
    "cost": 68.56,
    "intelligence": 10.1,
    "speed": 76.36
   },
   "calibration": {
    "score": 47.44
   },
   "capability": 28.77,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.3,
     5.7
    ],
    "B": [
     0.21,
     4.67
    ],
    "C": [
     0.21,
     3.96
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies",
    "kind": "estimate",
    "usd_per_1000": 0.011167487684729064
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "JevAct (einptein, jev1-2b-v2)",
   "endpoint_kind": "demo",
   "intelligence": {
    "I_open": 7.46,
    "I_sealed": 7.6,
    "base": 10.1,
    "per_type_split": {
     "open|choice": {
      "cc": 22.3825,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.9459,
       "hard": 9.4055,
       "judge": 40.6724,
       "standard": 17.1201
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0,
       "hard": -30.4348,
       "judge": -40,
       "standard": -33.3333
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 34.0168,
       "hard": -40.2474,
       "judge": 11.2946,
       "standard": -47.936
      }
     },
     "sealed|choice": {
      "cc": 22.8059,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 22.1501,
       "hard": 22.4389,
       "judge": 18.593,
       "standard": 30.1875
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 29.4118,
       "hard": 9.0909,
       "judge": -10,
       "standard": -25
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 2.6763,
       "hard": -73.6675,
       "judge": -35.3444,
       "standard": -0.6115
      }
     }
    }
   },
   "jevbench_score": 1.11,
   "key": "jevact",
   "last_measured_on": "2026-10-06",
   "licence": "not stated (the author published only an endpoint and the evaluation client)",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 43,
    "axes": {
     "calibration": 47.44,
     "cost": 68.56,
     "intelligence": 10.1,
     "speed": 76.45
    },
    "board_release": "v1.6.43",
    "capability": 28.77,
    "capability_api_board_rank": 29,
    "capability_open_board_rank": null,
    "capability_rank": 160,
    "composite_ci95": {
     "A": [
      0.3,
      5.7
     ],
     "B": [
      0.21,
      4.67
     ],
     "C": [
      0.21,
      3.96
     ]
    },
    "jevbench_score": 1.11,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 198,
    "ranks": {
     "A": 198,
     "B": 198,
     "C": 198,
     "capability": 231
    },
    "scores": {
     "A": 1.11,
     "B": 0.83,
     "C": 0.77
    },
    "speed": {
     "adjustment": "x2 (demo)",
     "n": 579,
     "p50_s_adjusted": 0.801,
     "p50_s_raw": 0.4005,
     "p95_s_adjusted": 2.829,
     "p95_s_raw": 1.4145
    },
    "views": null
   },
   "rank": 239,
   "ranked": true,
   "ranks": {
    "A": 239,
    "B": 239,
    "C": 239,
    "capability": 272
   },
   "repo": "https://github.com/fstandhartinger/jevbench/issues/66",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 1.11,
    "B": 0.83,
    "C": 0.77
   },
   "speed": {
    "adjustment": "x2 (demo)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.808,
    "p50_s_raw": 0.404,
    "p95_s_adjusted": 2.861,
    "p95_s_raw": 1.431
   },
   "status": {
    "answered_ok": 579,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "67145bec630ccceb1b6cbf2ed3545a5fd7ece3aa60a4e43e06f395b75da7c9f5"
   }
  },
  {
   "adapter_id": "images/jevk5-v0.3-4b/spec.json:jevk5_v15.JevK5Warm",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 89.80312361577623,
    "cost": 77.61552273233744,
    "intelligence": 38.53201254374527,
    "speed": 93.8287164149578
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05458773337668626,
      "ece_score": 89.08245332466275,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.84490011579697,
      "score_before_time_limit": 89.08245332466275,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "noul": {
      "abstention_rate": 0.33692722371967654,
      "brier": 0.11477714806826486,
      "ece": 0.053786553580231086,
      "n": 371,
      "score": 89.24268928395378,
      "score_before_time_limit": 89.24268928395378,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9566395663956639,
      "interval90_width": 0.4780939476061429,
      "n": 369,
      "nrps": 0.09977232020092458,
      "score": 91.32178144757796,
      "score_before_time_limit": 91.32178144757796,
      "time_limit_failed_share": 0,
      "top_ece": 0.03689602542375804
     }
    },
    "score": 89.80312361577623
   },
   "capability": 64.16756807976074,
   "class": "unclassified",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 921,181 input + 0 output tokens over 1,624 decisions = 567.229679803 tokens/decision. GPU-time cost USD 0.00557377254813054/1,000; old list-price cost USD 0.0170168903940887/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00557377254813054/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005573772548130538
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JevK5 v0.3 (4B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 38.333624208131226,
    "I_sealed": 38.730400879359316,
    "base": 38.53201254374527,
    "excess": -2.970164313666333,
    "gap": -0.39677667122808913,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 60.58350447013268,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 60.36489462094999,
       "judge": 64.40342781806196,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 20.41614906832298,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 47.82608695652173,
       "judge": -9.999999999999998,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 34.00121908593804,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.18739024047374,
       "hard": 34.10450662572488,
       "judge": 37.96632221813109,
       "standard": 18.253903730806933
      }
     },
     "sealed|choice": {
      "cc": 58.82061791071595,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 68.07331313280615,
       "hard": 55.712669683257914,
       "judge": 59.031657355679705,
       "standard": 60.09360758714129
      }
     },
     "sealed|noul": {
      "cc": 22.94594773546748,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48,
       "hard": 0.9708737864077666,
       "judge": 39.47368421052631,
       "standard": 29.5774647887324
      }
     },
     "sealed|score": {
      "cc": 34.42463699189452,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 65.98468618250719,
       "hard": 27.333586499919292,
       "judge": 34.53681846177148,
       "standard": 32.6584411757232
      }
     }
    }
   },
   "jevbench_score": 39.181248303247145,
   "key": "jevk5-v0.3-4b",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9471544715447154,
    "decisive_rate": 0.6630727762803235,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.88230801873149,
     "cost": 63.07359395281681,
     "intelligence": 38.53201254374527,
     "speed": 93.9472917853559
    },
    "board_release": "v1.6.43",
    "capability": 64.20716028123837,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 40,
    "capability_rank": 62,
    "composite_ci95": {
     "A": [
      24.2437041451056,
      49.59495089694384
     ],
     "B": [
      20.90887497172086,
      45.03359602914599
     ],
     "C": [
      16.835905656323334,
      34.440938122877675
     ]
    },
    "jevbench_score": 37.36493548751909,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 56,
    "rank": 86,
    "ranks": {
     "A": 86,
     "B": 90,
     "C": 87,
     "capability": 104
    },
    "scores": {
     "A": 37.36493548751909,
     "B": 33.16714732620591,
     "C": 25.947871866332704
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.18763320203870534,
     "p50_s_raw": 0.018816601019352674,
     "p95_s_adjusted": 0.21476382557302715,
     "p95_s_raw": 0.03238191278651357
    },
    "views": {
     "Balanced 33:33:33": 33.96793409727579,
     "Emphasis on Accuracy 60:20:20": 28.454799805414766,
     "Emphasis on Cost 20:20:60": 35.283091257812984,
     "Emphasis on Speed 20:60:20": 40.26908605684886,
     "Intelligence only": 22.883638070578947
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 150,
   "ranked": true,
   "ranks": {
    "A": 150,
    "B": 151,
    "C": 152,
    "capability": 134
   },
   "repo": "https://huggingface.co/alibiserikbay/JevK5",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 39.181248303247145,
    "B": 34.2961409909316,
    "C": 27.209200210588296
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18882605116814374,
    "p50_s_raw": 0.019413025584071875,
    "p95_s_adjusted": 0.21931404622737313,
    "p95_s_raw": 0.03465702311368657
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.78876192812696,
      89.6881064689264
     ],
     "I": [
      32.336664057232596,
      43.498424875066746
     ],
     "capability": [
      59.02389559614775,
      65.70541220096806
     ],
     "composite": null,
     "seed": 48
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.80312361577623,
     "capability": 64.16756807976074,
     "intelligence": 38.53201254374527
    },
    "per_type": {
     "choice": {
      "accuracy": 0.688,
      "calibration": 88.84490011579697,
      "cc": 60.099656129515054,
      "cc_by_type": {
       "choice": 60.099656129515054
      },
      "cc_tiered": 59.70206119042432,
      "cc_tiered_open": 60.58350447013268,
      "cc_tiered_sealed": 58.82061791071595,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.099656129515054
     },
     "noul": {
      "accuracy": 0.6213333333333333,
      "calibration": 89.24268928395378,
      "cc": 24.26666666666666,
      "cc_by_type": {
       "noul": 24.26666666666666
      },
      "cc_tiered": 21.681048401895232,
      "cc_tiered_open": 20.41614906832298,
      "cc_tiered_sealed": 22.94594773546748,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.26666666666666
     },
     "score": {
      "accuracy": 0.552,
      "calibration": 91.32178144757796,
      "cc": 35.814600564516994,
      "cc_by_type": {
       "score": 35.814600564516994
      },
      "cc_tiered": 34.21292803891628,
      "cc_tiered_open": 34.00121908593804,
      "cc_tiered_sealed": 34.42463699189452,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.814600564516994
     }
    },
    "raw_on_A": {
     "calibration": 86.97080976439989,
     "capability": 64.96464026185238,
     "intelligence": 42.95847075930487
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 35.99734346014853,
    "Emphasis on Accuracy 60:20:20": 29.284605157009096,
    "Emphasis on Cost 20:20:60": 39.45447951367246,
    "Emphasis on Speed 20:60:20": 41.935430880750744,
    "Intelligence only": 22.883638070578947
   }
  },
  {
   "adapter_id": "images/jevk5-v02/spec.json:jevk5_direct.JevK5DirectAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "allebee",
   "axes": {
    "calibration": 89.38710082819252,
    "cost": 77.61552273233744,
    "intelligence": 36.261123811123895,
    "speed": 91.22516918010956
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05722074836783767,
      "ece_score": 88.55585032643246,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.55585032643246,
      "score_before_time_limit": 88.55585032643246,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3800539083557951,
      "brier": 0.12953478188668033,
      "ece": 0.0506873602652232,
      "n": 371,
      "score": 89.86252794695537,
      "score_before_time_limit": 89.86252794695537,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.981029810298103,
      "interval90_width": 0.5517163504968384,
      "n": 369,
      "nrps": 0.098178280103309,
      "score": 89.74292421118973,
      "score_before_time_limit": 89.74292421118973,
      "time_limit_failed_share": 0,
      "top_ece": 0.053481617836448225
     }
    },
    "score": 89.38710082819252
   },
   "capability": 62.824112319658205,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 921,181 input + 0 output tokens over 1,624 decisions = 567.229679803 tokens/decision. GPU-time cost USD 0.00557377254813054/1,000; old list-price cost USD 0.0170168903940887/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00557377254813054/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005573772548130538
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JevK5 v0.2.0",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 39.85943426682522,
    "I_sealed": 32.66281335542257,
    "base": 36.261123811123895,
    "excess": 4.623233268964404,
    "gap": 7.196620911402647,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 56.2584494703768,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 57.189189189189186,
       "hard": 52.815350739226176,
       "judge": 56.49307844429796,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 29.9399585921325,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 47.82608695652173,
       "judge": -9.999999999999998,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 33.37989473796636,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.905656531340256,
       "hard": 27.673703730451283,
       "judge": 40.84579963079196,
       "standard": 20.83053851707115
      }
     },
     "sealed|choice": {
      "cc": 55.096468823579634,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 64.08247727440693,
       "hard": 55.20361990950226,
       "judge": 51.58286778398511,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 6.636284338632712,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": -14.563106796116509,
       "judge": 18.421052631578938,
       "standard": 12.676056338028175
      }
     },
     "sealed|score": {
      "cc": 36.25568690405537,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 64.58927438023056,
       "hard": 27.57268337567902,
       "judge": 41.004484216055225,
       "standard": 32.331704254720705
      }
     }
    }
   },
   "jevbench_score": 33.60100126653793,
   "key": "jevk5-v02",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (code/adapter); Apache-2.0 (Qwen3.5-4B base)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9347826086956522,
    "decisive_rate": 0.6199460916442049,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.38710082819252,
     "cost": 63.07359395281681,
     "intelligence": 36.261123811123895,
     "speed": 91.56570599843724
    },
    "board_release": "v1.6.43",
    "capability": 62.824112319658205,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 47,
    "capability_rank": 72,
    "composite_ci95": {
     "A": [
      21.332485805393638,
      41.48362112591029
     ],
     "B": [
      18.26634005992914,
      37.259117524916284
     ],
     "C": [
      14.814226253745582,
      28.808070226326592
     ]
    },
    "jevbench_score": 32.09901620353128,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 66,
    "rank": 97,
    "ranks": {
     "A": 97,
     "B": 99,
     "C": 99,
     "capability": 114
    },
    "scores": {
     "A": 32.09901620353128,
     "B": 28.24080105110956,
     "C": 22.290983474674498
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23078905596630647,
     "p50_s_raw": 0.04039452798315324,
     "p95_s_adjusted": 0.30214414960937575,
     "p95_s_raw": 0.07607207480468788
    },
    "views": {
     "Balanced 33:33:33": 29.02935442351159,
     "Emphasis on Accuracy 60:20:20": 24.01393367218798,
     "Emphasis on Cost 20:20:60": 30.556198079902455,
     "Emphasis on Speed 20:60:20": 34.51299399305244,
     "Intelligence only": 19.071452492870637
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 158,
   "ranked": true,
   "ranks": {
    "A": 158,
    "B": 159,
    "C": 160,
    "capability": 146
   },
   "repo": "https://github.com/allebee/jevk5",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 33.60100126653793,
    "B": 29.158180683740813,
    "C": 23.334028657318008
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23864530100254341,
    "p50_s_raw": 0.04432265050127171,
    "p95_s_adjusted": 0.31603137055644764,
    "p95_s_raw": 0.08301568527822382
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.43880788393268,
      90.09857764344801
     ],
     "I": [
      30.721600512376973,
      40.44080848333523
     ],
     "capability": [
      58.00843038572983,
      64.2177747361447
     ],
     "composite": null,
     "seed": 49
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.38710082819252,
     "capability": 62.824112319658205,
     "intelligence": 36.261123811123895
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6546666666666666,
      "calibration": 88.55585032643246,
      "cc": 55.836798878394866,
      "cc_by_type": {
       "choice": 55.836798878394866
      },
      "cc_tiered": 55.67745914697822,
      "cc_tiered_open": 56.2584494703768,
      "cc_tiered_sealed": 55.096468823579634,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 55.836798878394866
     },
     "noul": {
      "accuracy": 0.5733333333333334,
      "calibration": 89.86252794695537,
      "cc": 14.666666666666671,
      "cc_by_type": {
       "noul": 14.666666666666671
      },
      "cc_tiered": 18.288121465382606,
      "cc_tiered_open": 29.9399585921325,
      "cc_tiered_sealed": 6.636284338632712,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 14.666666666666671
     },
     "score": {
      "accuracy": 0.5546666666666666,
      "calibration": 89.74292421118973,
      "cc": 36.384921485746844,
      "cc_by_type": {
       "score": 36.384921485746844
      },
      "cc_tiered": 34.81779082101087,
      "cc_tiered_open": 33.37989473796636,
      "cc_tiered_sealed": 36.25568690405537,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.384921485746844
     }
    },
    "raw_on_A": {
     "calibration": 85.60375573005521,
     "capability": 62.620633576812004,
     "intelligence": 39.637511423568796
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 30.68321401534137,
    "Emphasis on Accuracy 60:20:20": 24.674042084849322,
    "Emphasis on Cost 20:20:60": 34.067631959861544,
    "Emphasis on Speed 20:60:20": 35.85317631589842,
    "Intelligence only": 19.071452492870637
   }
  },
  {
   "adapter_id": "images/jevone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Juspay",
   "axes": {
    "calibration": 91.76581828544445,
    "cost": 72.79744483400026,
    "intelligence": 45.955077428728934,
    "speed": 89.58480866139566
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.034452812499999874,
      "ece_score": 93.10943750000003,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 93.10943750000003,
      "score_before_time_limit": 93.10943750000003,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.376,
      "brier": 0.10284091864000006,
      "ece": 0.03908880000000003,
      "n": 375,
      "score": 92.18224,
      "score_before_time_limit": 92.18224,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9706666666666667,
      "interval90_width": 0.3935555555555556,
      "n": 375,
      "nrps": 0.08401831953999996,
      "score": 90.00577735633334,
      "score_before_time_limit": 90.00577735633334,
      "time_limit_failed_share": 0,
      "top_ece": 0.05793306666666674
     }
    },
    "score": 91.76581828544445
   },
   "capability": 68.8604478570867,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 821,899 input + 1,624 output tokens over 1,624 decisions = 507.09544335 tokens/decision. GPU-time cost USD 0.00806773898124576/1,000; old list-price cost USD 0.101219088669951/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00806773898124576/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.008067738981245761
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JevOne",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 47.580446331109215,
    "I_sealed": 44.329708526348654,
    "base": 45.955077428728934,
    "excess": 0.6773501623223175,
    "gap": 3.2507378047605613,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 80.11686099440308,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 77.35136835482857,
       "judge": 84.17930125247197,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 20.761214630779854,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 30.434782608695656,
       "judge": -9.999999999999998,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 41.86326336814473,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.82060606060606,
       "hard": 35.2175,
       "judge": 43.91203389830509,
       "standard": 46.102962962962955
      }
     },
     "sealed|choice": {
      "cc": 71.7077246661197,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 82.70637794693668,
       "hard": 65.89366515837104,
       "judge": 73.9292364990689,
       "standard": 74.50424929178472
      }
     },
     "sealed|noul": {
      "cc": 13.421236874491711,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48,
       "hard": -8.737864077669899,
       "judge": 13.157894736842103,
       "standard": 40.8450704225352
      }
     },
     "sealed|score": {
      "cc": 47.86016403843456,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.93652482269504,
       "hard": 43.95980392156864,
       "judge": 49.40958083832332,
       "standard": 45.29857868020299
      }
     }
    }
   },
   "jevbench_score": 58.70681958482434,
   "key": "jevone",
   "last_measured_on": "2026-10-02",
   "licence": "Component-specific JevOne/Qwen/SGLang terms recorded in RESULT.md",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9572649572649573,
    "decisive_rate": 0.624,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.76581828544445,
     "cost": 39.84212731591276,
     "intelligence": 45.955077428728934,
     "speed": 89.83885080776159
    },
    "board_release": "v1.6.43",
    "capability": 68.8604478570867,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      23.186488200616,
      37.838876804389514
     ],
     "B": [
      21.60212409324276,
      36.562829588228304
     ],
     "C": [
      16.101727917094447,
      27.301863296622916
     ]
    },
    "jevbench_score": 31.14510331640707,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 67,
    "rank": 98,
    "ranks": {
     "A": 98,
     "B": 96,
     "C": 100,
     "capability": 85
    },
    "scores": {
     "A": 31.14510331640707,
     "B": 29.58577784402432,
     "C": 21.62854396972713
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2735971907153726,
     "p50_s_raw": 0.06179859535768628,
     "p95_s_adjusted": 0.3793178545311093,
     "p95_s_raw": 0.11465892726555464
    },
    "views": {
     "Balanced 33:33:33": 27.74831039177889,
     "Emphasis on Accuracy 60:20:20": 26.4196991707309,
     "Emphasis on Cost 20:20:60": 24.78908517771933,
     "Emphasis on Speed 20:60:20": 33.418202927958156,
     "Intelligence only": 24.64935315350002
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 113,
   "ranked": true,
   "ranks": {
    "A": 113,
    "B": 113,
    "C": 117,
    "capability": 108
   },
   "repo": "https://huggingface.co/juspay/jev-one",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 58.70681958482434,
    "B": 53.2510808658194,
    "C": 40.76862471168357
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2804038129746914,
    "p50_s_raw": 0.0652019064873457,
    "p95_s_adjusted": 0.3924056202173233,
    "p95_s_raw": 0.12120281010866166
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.6-35B-A3B BF16 with JevOne bidirectional option-logit mapping",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.59503348121109,
      91.11403729704442
     ],
     "I": [
      40.574264452604204,
      50.931916001637504
     ],
     "capability": [
      63.974665483283545,
      69.89087834774071
     ],
     "composite": null,
     "seed": 50
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.76581828544445,
     "capability": 68.8604478570867,
     "intelligence": 45.955077428728934
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7946666666666666,
      "calibration": 93.10943750000003,
      "cc": 73.74079933309964,
      "cc_by_type": {
       "choice": 73.74079933309964
      },
      "cc_tiered": 75.91229283026139,
      "cc_tiered_open": 80.11686099440308,
      "cc_tiered_sealed": 71.7077246661197,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.74079933309964
     },
     "noul": {
      "accuracy": 0.5973333333333334,
      "calibration": 92.18224,
      "cc": 19.466666666666676,
      "cc_by_type": {
       "noul": 19.466666666666676
      },
      "cc_tiered": 17.091225752635783,
      "cc_tiered_open": 20.761214630779854,
      "cc_tiered_sealed": 13.421236874491711,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 19.466666666666676
     },
     "score": {
      "accuracy": 0.6,
      "calibration": 90.00577735633334,
      "cc": 46.965815789473666,
      "cc_by_type": {
       "score": 46.965815789473666
      },
      "cc_tiered": 44.861713703289645,
      "cc_tiered_open": 41.86326336814473,
      "cc_tiered_sealed": 47.86016403843456,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 46.965815789473666
     }
    },
    "raw_on_A": {
     "calibration": 86.03357993512108,
     "capability": 69.68729793470257,
     "intelligence": 53.34101593428407
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 54.31327526677244,
    "Emphasis on Accuracy 60:20:20": 46.83649418036129,
    "Emphasis on Cost 20:20:60": 56.97496969082264,
    "Emphasis on Speed 20:60:20": 61.22695500801212,
    "Intelligence only": 38.82044396643135
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "kadirbekovvv",
   "axes": {
    "calibration": 55.43211615661533,
    "cost": null,
    "intelligence": 18.416504152111088,
    "speed": 90.40620976128842
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.26331740063992715,
      "ece_score": 47.33651987201457,
      "mean_tvd": null,
      "n": 729,
      "n_ece_hard": 296,
      "n_tvd": 0,
      "score": 47.273404512185216,
      "score_before_time_limit": 47.33651987201457,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.12568306010928965,
      "brier": 0.2429692790949114,
      "ece": 0.21804147724652553,
      "n": 366,
      "score": 56.39170455069489,
      "score_before_time_limit": 56.39170455069489,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8375350140056023,
      "interval90_width": 0.6150793650793649,
      "n": 357,
      "nrps": 0.19607308788933592,
      "score": 62.631239406965896,
      "score_before_time_limit": 62.631239406965896,
      "time_limit_failed_share": 0,
      "top_ece": 0.2756510619856731
     }
    },
    "score": 55.43211615661533
   },
   "capability": 36.92431015436321,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 3948e3f1..., and the later re-freezes, which only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base (the module RAISES 'base model lacks an explicit market-listing decision'; the artifact's own metadata names this base); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens as reported by the author's in-process adapter (input 681,813, output_tokens 0).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jevora-9b-v1 (kadirbekovvv / doxanocap, Qwen3.5-9B-Base derivative)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's in-process adapter and native decision head (no server).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 21.935398847280897,
    "I_sealed": 14.897609456941275,
    "base": 18.416504152111088,
    "excess": -2.801737380016089,
    "gap": 7.037789390339622,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 37.29135872265705,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 57.189189189189186,
       "hard": 30.166719094054738,
       "judge": 48.58272907053395,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 28.514837819185644,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -4.347826086956519,
       "judge": 39.99999999999999,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 39.56118625108944,
       "hard": 3.2474489744614754,
       "judge": -25.16569669784634,
       "standard": -22.34160066754731
      }
     },
     "sealed|choice": {
      "cc": 27.18704645330748,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 54.2566709021601,
       "hard": 21.938244477853576,
       "judge": 21.547502448579845,
       "standard": 32.609154186880424
      }
     },
     "sealed|noul": {
      "cc": 17.50578191751635,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 51.72413793103448,
       "hard": 5.454545454545445,
       "judge": 16.92307692307693,
       "standard": 25.373134328358216
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 2.429989968802293,
       "hard": -40.50178363761847,
       "judge": -25.860796025760123,
       "standard": 10.253044034947001
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "jevora-9b-v1",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "listed",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "kadirbekovvv/jevora-9b-v1@c644b9c7a3b67b7acc3279523e3bee69a4be473d",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base and raises rather than estimating; one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.728125,
    "decisive": 320,
    "decisive_rate": 0.8743169398907104,
    "n": 375,
    "supported": true,
    "valid": 366
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 55.45315460989178,
     "cost": null,
     "intelligence": 18.54232988347315,
     "speed": 90.58228710164383
    },
    "board_release": "v1.6.43",
    "capability": 36.99774224668246,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.27591127399355175,
     "p50_s_raw": 0.06295563699677587,
     "p95_s_adjusted": 0.3169580915942788,
     "p95_s_raw": 0.0834790457971394
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/kadirbekovvv/jevora-9b-v1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27972743902355435,
    "p50_s_raw": 0.06486371951177716,
    "p95_s_adjusted": 0.3255696814507246,
    "p95_s_raw": 0.0877848407253623
   },
   "status": {
    "answered_ok": 1452,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5,
       "cc": -46.83461706671449,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": 50,
        "score": -223.83718453347683
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.33663366336633666,
       "cc": -3.5284110215130844,
       "cc_by_type": {
        "choice": 10.92258748674444,
        "noul": 43.99999999999999,
        "score": -65.50782055128369
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 3.1172240518339493,
       "cc_by_type": {
        "choice": 28.45528455284553,
        "noul": 33.33333333333333,
        "score": -52.43694573067701
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 3.1172240518339493
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 53.62620746968167,
       "cc_by_type": {
        "choice": 71.42857142857143,
        "noul": 19.999999999999996,
        "score": 69.45005098047356
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 53.62620746968167
      },
      "jaggedness": {
       "accuracy": 0.37777777777777777,
       "cc": -6.695722360583766,
       "cc_by_type": {
        "choice": 22.040423484119355,
        "noul": -13.513513513513509,
        "score": -28.614077052357146
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.6741214057507987,
       "cc": 51.730031018571616,
       "cc_by_type": {
        "choice": 47.697455313903326,
        "noul": 73.1958762886598,
        "score": 34.2967614531517
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 51.730031018571616
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2222222222222222,
       "cc": -38.364547830866904,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -50,
        "score": -66.12457132765226
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.4727272727272727,
       "cc": 5.811727778234681,
       "cc_by_type": {
        "choice": 35.441799804369104,
        "noul": 8.333333333333325,
        "score": -26.339949802998387
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 5.811727778234681
      },
      "negation_pair_gen": {
       "accuracy": 0.3488372093023256,
       "cc": -24.610571890241573,
       "cc_by_type": {
        "choice": 10.000000000000002,
        "noul": -13.33333333333333,
        "score": -70.49838233739139
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.2708333333333333,
       "cc": 6.790945406125167,
       "cc_by_type": {
        "choice": 6.790945406125167
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 6.790945406125167
      },
      "perturb_pair": {
       "accuracy": 0.6875,
       "cc": 59.76765762955464,
       "cc_by_type": {
        "choice": 48.94894894894895,
        "noul": 100,
        "score": 30.354023939714992
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 59.76765762955464
      },
      "usecase": {
       "accuracy": 0.38636363636363635,
       "cc": 1.345314494671392,
       "cc_by_type": {
        "choice": 25.245098039215712,
        "noul": -12.5,
        "score": -8.709154555201536
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 1.345314494671392
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -17.73475850312556,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0,
        "score": -77.01379931890048
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 38.634690940151195,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": -63.358599990606315
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 38.634690940151195
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 58.46529563147624,
       "cc_by_type": {
        "choice": 44.24778761061947,
        "noul": 100,
        "score": 31.14809928380925
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 58.46529563147624
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 16.196083473787407,
       "cc_by_type": {
        "choice": -0.5586592178770929,
        "noul": 0,
        "score": 49.14690963923931
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 16.196083473787407
      },
      "en": {
       "accuracy": 0.46481320590790615,
       "cc": 13.665211396900302,
       "cc_by_type": {
        "choice": 28.167828071417194,
        "noul": 25.899280575539564,
        "score": -13.07147445625585
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 13.665211396900302
      },
      "es": {
       "accuracy": 0.6451612903225806,
       "cc": 38.89156095653784,
       "cc_by_type": {
        "choice": 41.27243066884176,
        "noul": 100,
        "score": -24.59774779922823
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 38.89156095653784
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -43.89799635701275,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": -100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5,
       "cc": 20.45477668736496,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 3.717271238565456
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 20.45477668736496
      },
      "hi": {
       "accuracy": 0.09523809523809523,
       "cc": -121.9593465935224,
       "cc_by_type": {
        "choice": -17.31843575418994,
        "noul": -75,
        "score": -273.5596040263772
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 65.42216278743965,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 10.296106444937813
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 65.42216278743965
      },
      "ja": {
       "accuracy": 0.48,
       "cc": -0.003854126466692994,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": -19.999999999999996,
        "score": -8.526256581147496
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 19.542302638686518,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 3.0713523605039983
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 19.542302638686518
      },
      "mixed": {
       "accuracy": 0.6428571428571429,
       "cc": 49.22501272004511,
       "cc_by_type": {
        "choice": 50,
        "noul": 33.33333333333333,
        "score": 64.34170482680202
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 49.22501272004511
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 33.90898539003356,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 100,
        "score": -46.324991881847396
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 33.90898539003356
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 22.898515881046155,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 19.999999999999996,
        "score": -28.452145966679552
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 22.898515881046155
      },
      "pt": {
       "accuracy": 0.5357142857142857,
       "cc": 38.58246838017991,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": 100,
        "score": -10.00392072864118
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 38.58246838017991
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 6.878374932822606,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": 19.999999999999996,
        "score": 28.294699266552925
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 6.878374932822606
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -33.32266237154819,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": 0,
        "score": -91.27233494073153
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.2857142857142857,
       "cc": -44.56124096628158,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -69.16759386658669
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.3,
       "cc": -59.687737369534545,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336,
        "score": -169.45869233459234
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.46,
       "cc": 30.251248588759843,
       "cc_by_type": {
        "choice": 30.251248588759843
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 30.251248588759843
      },
      "noul": {
       "accuracy": 0.6213333333333333,
       "cc": 24.26666666666666,
       "cc_by_type": {
        "noul": 24.26666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 24.26666666666666
      },
      "score": {
       "accuracy": 0.352,
       "cc": -15.561798891550138,
       "cc_by_type": {
        "score": -15.561798891550138
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.18181818181818182,
       "cc": -53.49933930121218,
       "cc_by_type": {
        "choice": 4,
        "noul": -50,
        "score": -114.49801790363652
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -27.327813582184664,
       "cc_by_type": {
        "choice": -11.801242236024848,
        "noul": -33.333333333333336,
        "score": -36.84886517719581
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 67.16319530556196,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 84.0099111199379
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 67.16319530556196
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 28.345389256969124,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 33.33333333333333,
        "score": 33.5741794668138
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 28.345389256969124
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": -3.2842516228736685,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": -33.333333333333336,
        "score": -4.0556534193456395
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.21052631578947367,
       "cc": -61.25795271724025,
       "cc_by_type": {
        "choice": -13.207547169811319,
        "noul": -66.66666666666667,
        "score": -103.89964431524277
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.43333333333333335,
       "cc": -1.4206709993739324,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -14.28571428571429,
        "score": -40.956690869270254
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.13636363636363635,
       "cc": -8.466083304599605,
       "cc_by_type": {
        "choice": -25.435540069686407,
        "noul": 0,
        "score": 0.03729015588759266
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 7.990436824383561,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -19.999999999999996,
        "score": 48.137977139817345
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 7.990436824383561
      },
      "knowledge_graphs": {
       "accuracy": 0.18181818181818182,
       "cc": -21.352245247704335,
       "cc_by_type": {
        "choice": -28.479657387580303,
        "noul": -33.333333333333336,
        "score": -2.2437450221993727
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.56,
       "cc": 3.569853000991013,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 11.111111111111116,
        "score": -77.1457381546497
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 3.569853000991013
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 16.028989510323616,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 33.33333333333333,
        "score": -52.98830028623345
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 16.028989510323616
      },
      "llm_guardrails": {
       "accuracy": 0.3333333333333333,
       "cc": -12.070180855485154,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -19.999999999999996,
        "score": -47.11265389083166
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.7050691244239631,
       "cc": 53.53856111416466,
       "cc_by_type": {
        "choice": 57.92207792207793,
        "noul": 87.87878787878789,
        "score": 14.814817541628155
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 53.53856111416466
      },
      "moderation_trust_safety": {
       "accuracy": 0.3333333333333333,
       "cc": 39.592028247842826,
       "cc_by_type": {
        "choice": -3.646833013435703,
        "noul": 100,
        "score": 22.42291775696419
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 39.592028247842826
      },
      "recruiting": {
       "accuracy": 0.5555555555555556,
       "cc": 17.99421275407803,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 20.742155439578582
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 17.99421275407803
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -38.314615283872065,
       "cc_by_type": {
        "choice": 26.10837438423645,
        "noul": -50,
        "score": -91.05222023585264
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.34782608695652173,
       "cc": 6.055643005647522,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": 0,
        "score": 5.123450756073
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 6.055643005647522
      },
      "search_retrieval": {
       "accuracy": 0.75,
       "cc": 67.96574075461801,
       "cc_by_type": {
        "choice": 57.295373665480426,
        "noul": 100,
        "score": 46.601848598373564
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 67.96574075461801
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      49.71252057656553,
      59.65793702420633
     ],
     "I": [
      13.85411913413391,
      23.55473519133409
     ],
     "capability": [
      32.36265845391065,
      41.127021044667465
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 37
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 55.43211615661533,
     "capability": 36.92431015436321,
     "intelligence": 18.416504152111088
    },
    "per_type": {
     "choice": {
      "accuracy": 0.46,
      "calibration": 47.273404512185216,
      "cc": 30.251248588759843,
      "cc_by_type": {
       "choice": 30.251248588759843
      },
      "cc_tiered": 32.23920258798226,
      "cc_tiered_open": 37.29135872265705,
      "cc_tiered_sealed": 27.18704645330748,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 30.251248588759843
     },
     "noul": {
      "accuracy": 0.6213333333333333,
      "calibration": 56.39170455069489,
      "cc": 24.26666666666666,
      "cc_by_type": {
       "noul": 24.26666666666666
      },
      "cc_tiered": 23.010309868350998,
      "cc_tiered_open": 28.514837819185644,
      "cc_tiered_sealed": 17.50578191751635,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.26666666666666
     },
     "score": {
      "accuracy": 0.352,
      "calibration": 62.631239406965896,
      "cc": -15.561798891550138,
      "cc_by_type": {
       "score": -15.561798891550138
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 58.65667414349311,
     "capability": 37.50098153070991,
     "intelligence": 16.345288917926695
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.029333333333333322,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.02400000000000002,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.04800000000000004,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "pragmaticfrontier (Hugging Face)",
   "axes": {
    "calibration": 83.15885665129233,
    "cost": 83.31206101384694,
    "intelligence": 25.626746881681953,
    "speed": 90.75287985569767
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09529471475409836,
      "ece_score": 80.94105704918033,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.94105704918033,
      "score_before_time_limit": 80.94105704918033,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.6595174262734584,
      "brier": 0.15627444447721173,
      "ece": 0.07921313672922253,
      "n": 373,
      "score": 84.1573726541555,
      "score_before_time_limit": 84.1573726541555,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9838709677419355,
      "interval90_width": 0.7598566308243728,
      "n": 372,
      "nrps": 0.12251524337627304,
      "score": 84.37814025054118,
      "score_before_time_limit": 84.37814025054118,
      "time_limit_failed_share": 0,
      "top_ece": 0.09496097580645166
     }
    },
    "score": 83.15885665129233
   },
   "capability": 54.39280176648714,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 4B active parameters; throughput 80585.75 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 712,537 input + 0 output tokens over 1,485 decisions = 479.822895623 tokens/decision. GPU-time cost USD 0.00359968611509844/1,000; old list-price cost USD 0.00959645791245791/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00359968611509844/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.009500493333333334
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003599686115098442
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jez-large-1.0.0 (pragmaticfrontier, Gemma-4-E4B label-logit decision model, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, Finland, EU, pay-as-you-go non-spot, USD 2.525/h), shared by the two Jez rows as two separate back-to-back uses (sealed input shredded from the node between them). The author's own server as shipped (torch backend, no vLLM serving) in a pre-built image on the stock vllm/vllm-openai:v0.30.0 base (torch 2.13.0+cu130, transformers 5.19.0, tokenizers 0.23.3, flash-linear-attention 0.5.2), container with --network none (cut proven from inside before the sealed input), Hub offline, every credential variable empty, read-only bundle, all 14 files of the pinned revision (15,516,502,246 bytes) sha256-verified inside the container against manifest 11092a9e... and all 169 author code files verified; our first-party loopback proxy only maps the author's over-limit 400 to 422; driven by the unchanged pinned pod_driver_v16, one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 22 Oct 2026 (05:35Z start of this row) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 30.648346345077897,
    "I_sealed": 20.605147418286013,
    "base": 25.626746881681953,
    "excess": 0.20367215643617342,
    "gap": 10.043198926791884,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 63.31336811872398,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 50.92796476879522,
       "judge": 72.31377719182598,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": -47.82608695652174,
       "judge": -40,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 28.63167091650972,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 44.0953,
       "hard": 17.99087037037035,
       "judge": 39.276420338983044,
       "standard": 26.214333333333318
      }
     },
     "sealed|choice": {
      "cc": 46.769936658102864,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 67.5984752223634,
       "hard": 38.591418989244815,
       "judge": 41.82174338883449,
       "standard": 60.13499261759124
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -6.896551724137934,
       "hard": -54.54545454545454,
       "judge": -53.84615384615385,
       "standard": -22.388059701492537
      }
     },
     "sealed|score": {
      "cc": 15.045505596755174,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 33.295781578947384,
       "hard": 3.934931267605657,
       "judge": 16.076345833333317,
       "standard": 26.595255909090877
      }
     }
    }
   },
   "jevbench_score": 14.186361224326426,
   "key": "jez-large-1.0.0",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (HF card metadata; the card also links the Gemma 4 licence terms for the base; the HF repo has no LICENSE file; author code repo is MIT), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "pragmaticfrontier/jez-large-1.0.0@c6f511cf031cdcf8119fd67954e2fcbc6d3f2dbf",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9763779527559056,
    "decisive": 127,
    "decisive_rate": 0.34048257372654156,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 208,
   "ranked": true,
   "ranks": {
    "A": 208,
    "B": 209,
    "C": 208,
    "capability": 200
   },
   "repo": "https://huggingface.co/pragmaticfrontier/jez-large-1.0.0",
   "scores": {
    "A": 14.186361224326426,
    "B": 11.614226981395028,
    "C": 9.851639739115571
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26939871499998846,
    "p50_s_raw": 0.05969935749999422,
    "p95_s_adjusted": 0.31211633210007217,
    "p95_s_raw": 0.0810581660500361
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author server pragmaticfrontier/jez at bf7c158169e5ff1db75a4b3ef3e1161cd5f50e91 (MIT; python3 -m jez.server, torch backend, --no-auth), started directly as shipped from an unchanged source copy (independent static review PASS WITH FINDINGS; no subprocess, eval/exec, pickle or trust_remote_code in the reviewed server closure; no branch on item text, task id or benchmark name). The review findings were the runtime-download fallbacks in the author code (hf_hub_download for the calibration file and from_pretrained base resolution when the checkpoint is a Hub id); they were never reached: the merged checkpoint was loaded from the local pinned /model with its own /model/calibration.json, Hub offline, container --network none, every credential variable empty; the owner cleared the code for one offline measurement on those terms. Author settings kept: order-debiased scoring (JEZ_DEBIAS=1, given and reverse order averaged), JEZ_MAX_BATCH=16, the checkpoint's calibration temperatures unchanged; no score fitting, no prompt transformation. FIRST MEASUREMENT (found by our own intake of new public Hugging Face decision models; not a free-queue submission, not in the Decision Index at pick-up). CONTEXT CAP: the author server refuses a question whose state+question exceeds 32,000 tokens and a request above 64,000 tokens (HTTP 400 max_tokens_exceeded); our reviewed loopback status proxy maps exactly that answer to 422 and passes everything else through unchanged; on this draw that is exactly the 15 items of 76k-81k tokens, pre-registered by id and position; they count as wrong, like elf-1-12b, seb-9b and the NeoHorse rows on this draw. SAME FAMILY: jez-large-1.0.0 (Gemma-4-E4B base) and jez-small-1.0.0 (Qwen3.5-0.8B base) are two checkpoints of one author measured back to back on one VM. PROVENANCE, author statements (unverified): LoRA r64/alpha128 merged into google/gemma-4-E4B-it (card); trained on 278,999 gold-label examples from 20 public sources (reports/mix-v2-manifest.json, not audited by us), no teacher model; the card reports its own acceptance gate 2 (Tier 1 vs base) as not met and states no JevBench or Decision Index result. No rule against public items was found in the reviewed code. LATENCY: maximum single decision 0.118 s on this card; no decision came near the 10 s limit. Determinism probe on the first 50 positions, two passes: 50/50 bit-identical.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 47.728766666666665,
       "cc_by_type": {
        "choice": 75,
        "noul": 50,
        "score": 18.18629999999999
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 47.728766666666665
      },
      "calibration_gen": {
       "accuracy": 0.39603960396039606,
       "cc": -5.296255211647228,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": -52,
        "score": 2.919293749999985
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.4158415841584158,
       "cc": -8.80096407361083,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": -85.18518518518519,
        "score": 7.5627807692307485
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.5833333333333334,
       "cc": 35.68471666666667,
       "cc_by_type": {
        "choice": 100,
        "noul": -46.666666666666664,
        "score": 53.72081666666667
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 35.68471666666667
      },
      "jaggedness": {
       "accuracy": 0.3055555555555556,
       "cc": -17.76067531981394,
       "cc_by_type": {
        "choice": 22.040423484119355,
        "noul": -72.97297297297297,
        "score": -2.3494764705882076
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.6389776357827476,
       "cc": 42.72053255265422,
       "cc_by_type": {
        "choice": 57.99947169146781,
        "noul": 29.89690721649485,
        "score": 40.26521874999999
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 42.72053255265422
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -22.07230976926854,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -75,
        "score": 7.752142857142841
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5490909090909091,
       "cc": 16.860791594146995,
       "cc_by_type": {
        "choice": 59.895663514835356,
        "noul": -33.333333333333336,
        "score": 24.02004460093896
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 16.860791594146995
      },
      "negation_pair_gen": {
       "accuracy": 0.29069767441860467,
       "cc": -26.800501111111107,
       "cc_by_type": {
        "choice": 26.666666666666668,
        "noul": -100,
        "score": -7.068169999999996
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.7291666666666666,
       "cc": 65.37949400798935,
       "cc_by_type": {
        "choice": 65.37949400798935
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 65.37949400798935
      },
      "perturb_pair": {
       "accuracy": 0.75,
       "cc": 52.72739511819512,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 39.99999999999999,
        "score": 48.212215384615384
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 52.72739511819512
      },
      "usecase": {
       "accuracy": 0.45454545454545453,
       "cc": 4.262127848366023,
       "cc_by_type": {
        "choice": 54.65686274509807,
        "noul": -62.5,
        "score": 20.6295208
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 4.262127848366023
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": -22.651981684981692,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": -50,
        "score": -3.6702307692307867
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 18.107952222222217,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -12.342810000000014
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 18.107952222222217
      },
      "de": {
       "accuracy": 0.71875,
       "cc": 61.49717155316148,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75,
        "score": 41.34992173913044
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 61.49717155316148
      },
      "el": {
       "accuracy": 0.4,
       "cc": 8.02099897579143,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -100,
        "score": 57.58254999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 8.02099897579143
      },
      "en": {
       "accuracy": 0.48045178105994785,
       "cc": 8.592013966401177,
       "cc_by_type": {
        "choice": 48.15782087111456,
        "noul": -39.568345323741006,
        "score": 17.18656635182998
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 8.592013966401177
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 48.436651377027154,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75,
        "score": -19.902117647058827
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 48.436651377027154
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -0.8360408014571945,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 29.18586666666667
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 36.61608699690402,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 31.024731578947375
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 36.61608699690402
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": 0.5619364757914317,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -50,
        "score": -14.794637500000007
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0.5619364757914317
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 60.95263080206362,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 10.91712857142857
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 60.95263080206362
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 12.098056231577091,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -19.999999999999996,
        "score": 13.482413333333342
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 12.098056231577091
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -19.5858462962963,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": 30.13134999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 92.06537777777777,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 76.19613333333332
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 92.06537777777777
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -17.185083982683985,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 0.39279999999999315
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 22.790673838931582,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -40,
        "score": 31.224327906976722
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 22.790673838931582
      },
      "pt": {
       "accuracy": 0.6071428571428571,
       "cc": 23.36979110257823,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": -19.999999999999996,
        "score": 32.537187096774176
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 23.36979110257823
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": -18.99189822695035,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -100,
        "score": 19.62005
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 15.302973188405792,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0,
        "score": -32.351950000000016
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 15.302973188405792
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 50.6590188172043,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": 100,
        "score": 48.75124999999999
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 50.6590188172043
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -3.979430320150655,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -27.75749999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6173333333333333,
       "cc": 50.57310702462733,
       "cc_by_type": {
        "choice": 50.57310702462733
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 50.57310702462733
      },
      "noul": {
       "accuracy": 0.33066666666666666,
       "cc": -33.86666666666667,
       "cc_by_type": {
        "noul": -33.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.432,
       "cc": 18.55008041516254,
       "cc_by_type": {
        "score": 18.55008041516254
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 18.55008041516254
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -30.149688235294118,
       "cc_by_type": {
        "choice": 36,
        "noul": -100,
        "score": -26.449064705882353
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": -10.605294780662168,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -55.55555555555556,
        "score": 16.907373076923093
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 37.238112466124655,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 0,
        "score": 27.97449999999998
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 37.238112466124655
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -33.806594298245614,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": 3.8433750000000044
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": -11.968426539855066,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -100,
        "score": 7.572981250000021
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -0.74184968553462,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": -100,
        "score": 35.510299999999916
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 18.85952832722833,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -71.42857142857143,
        "score": 28.007156410256407
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 18.85952832722833
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -15.346991460889763,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100,
        "score": 16.676795652173904
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.13333333333333333,
       "cc": -16.61181296296297,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -60,
        "score": 14.331227777777766
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.18181818181818182,
       "cc": -23.690844495273936,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -66.66666666666667,
        "score": -1.6221409090908967
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 17.945389593610518,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": -1.202590909090917
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 17.945389593610518
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -57.1503943548387,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -42.41892499999997
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": -0.5887295940854571,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 32.78275555555553
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.7142857142857143,
       "cc": 53.92926226213023,
       "cc_by_type": {
        "choice": 73.11688311688313,
        "noul": 57.57575757575757,
        "score": 31.095146093749992
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 53.92926226213023
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 3.7569469609724897,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 57.3361
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 3.7569469609724897
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 44.48287790634003,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 16.82839230769233
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 44.48287790634003
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -11.3009231623684,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -100,
        "score": 10.43220588235293
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.2608695652173913,
       "cc": -18.42371289855073,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -66.66666666666667,
        "score": 12.84480333333331
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.75,
       "cc": 47.43235876880814,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 0,
        "score": 63.6493894736842
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 47.43235876880814
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.65436930601125,
      85.039795395276
     ],
     "I": [
      22.47448065934068,
      28.306745826814417
     ],
     "capability": [
      50.91698361027989,
      55.85159962168382
     ],
     "composite": null,
     "seed": 39
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.15885665129233,
     "capability": 54.39280176648714,
     "intelligence": 25.626746881681953
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6173333333333333,
      "calibration": 80.94105704918033,
      "cc": 50.57310702462733,
      "cc_by_type": {
       "choice": 50.57310702462733
      },
      "cc_tiered": 55.041652388413425,
      "cc_tiered_open": 63.31336811872398,
      "cc_tiered_sealed": 46.769936658102864,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 50.57310702462733
     },
     "noul": {
      "accuracy": 0.33066666666666666,
      "calibration": 84.1573726541555,
      "cc": -33.86666666666667,
      "cc_by_type": {
       "noul": -33.86666666666667
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.432,
      "calibration": 84.37814025054118,
      "cc": 18.55008041516254,
      "cc_by_type": {
       "score": 18.55008041516254
      },
      "cc_tiered": 21.83858825663245,
      "cc_tiered_open": 28.63167091650972,
      "cc_tiered_sealed": 15.045505596755174,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 18.55008041516254
     }
    },
    "raw_on_A": {
     "calibration": 82.45723821371578,
     "capability": 54.02535130252623,
     "intelligence": 25.59346439133668
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 12.701947646090284,
    "Emphasis on Accuracy 60:20:20": 9.376020226736248,
    "Emphasis on Cost 20:20:60": 15.263945463388593,
    "Emphasis on Speed 20:60:20": 15.62125802990717,
    "Intelligence only": 6.731942988264025
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "pragmaticfrontier (Hugging Face)",
   "axes": {
    "calibration": 81.90437649433692,
    "cost": 86.2098204316998,
    "intelligence": 4.560234849177464,
    "speed": 92.50829073456676
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09639829836065571,
      "ece_score": 80.72034032786885,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.72034032786885,
      "score_before_time_limit": 80.72034032786885,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.8927613941018767,
      "brier": 0.2317885124664879,
      "ece": 0.07312252010723858,
      "n": 373,
      "score": 85.37549597855228,
      "score_before_time_limit": 85.37549597855228,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8924731182795699,
      "interval90_width": 0.7674731182795698,
      "n": 372,
      "nrps": 0.1775305450703582,
      "score": 79.61729317658961,
      "score_before_time_limit": 79.61729317658961,
      "time_limit_failed_share": 0,
      "top_ece": 0.11506179569892477
     }
    },
    "score": 81.90437649433692
   },
   "capability": 43.23230567175719,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.8B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 707,876 input + 0 output tokens over 1,485 decisions = 476.684175084 tokens/decision. GPU-time cost USD 0.00288185849500294/1,000; old list-price cost USD 0.00476684175084175/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00288185849500294/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.004719173333333333
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00288185849500294
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Jez-small-1.0.0 (pragmaticfrontier, Qwen3.5-0.8B label-logit decision model, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, Finland, EU, pay-as-you-go non-spot, USD 2.525/h), shared by the two Jez rows as two separate back-to-back uses (sealed input shredded from the node between them). The author's own server as shipped (torch backend, no vLLM serving) in a pre-built image on the stock vllm/vllm-openai:v0.30.0 base (torch 2.13.0+cu130, transformers 5.19.0, tokenizers 0.23.3, flash-linear-attention 0.5.2), container with --network none (cut proven from inside before the sealed input), Hub offline, every credential variable empty, read-only bundle, all 11 files of the pinned revision (2,033,398,221 bytes) sha256-verified inside the container against manifest 26570461... and all 169 author code files verified; our first-party loopback proxy only maps the author's over-limit 400 to 422; driven by the unchanged pinned pod_driver_v16, one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 22 Oct 2026 (05:46Z start of this row) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 4.606378088993198,
    "I_sealed": 4.51409160936173,
    "base": 4.560234849177464,
    "excess": -9.747240290724243,
    "gap": 0.09228647963146841,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 13.819134266979594,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 33.837837837837846,
       "hard": 1.8559295375904445,
       "judge": 20.89650626235992,
       "standard": 17.12013394725827
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -73.91304347826086,
       "judge": -90,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.356584848484864,
       "hard": -7.228344444444423,
       "judge": -4.739777966101699,
       "standard": -23.675696296296287
      }
     },
     "sealed|choice": {
      "cc": 13.54227482808519,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 37.10292249047015,
       "hard": 5.285069966462341,
       "judge": 15.377081292850164,
       "standard": 15.524151022990946
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -62.06896551724138,
       "hard": -94.54545454545455,
       "judge": -87.6923076923077,
       "standard": -85.07462686567165
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 4.811904385964915,
       "hard": -20.03648563380276,
       "judge": -4.9763859375000274,
       "standard": -3.007532727272766
      }
     }
    }
   },
   "jevbench_score": 0.1310454200964957,
   "key": "jez-small-1.0.0",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (HF card metadata; the HF repo has no LICENSE file; author code repo is MIT), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "pragmaticfrontier/jez-small-1.0.0@4151ad0762ca95eae85b5a10c6dd6bfc10eb7627",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.85,
    "decisive": 40,
    "decisive_rate": 0.10723860589812333,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 262,
   "ranked": true,
   "ranks": {
    "A": 262,
    "B": 262,
    "C": 262,
    "capability": 234
   },
   "repo": "https://huggingface.co/pragmaticfrontier/jez-small-1.0.0",
   "scores": {
    "A": 0.1310454200964957,
    "B": 0.0878954444947552,
    "C": 0.0910037639558998
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23508084200002485,
    "p50_s_raw": 0.04254042100001243,
    "p95_s_adjusted": 0.23875566719984817,
    "p95_s_raw": 0.04437783359992409
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author server pragmaticfrontier/jez at bf7c158169e5ff1db75a4b3ef3e1161cd5f50e91 (MIT; python3 -m jez.server, torch backend, --no-auth), started directly as shipped from an unchanged source copy (independent static review PASS WITH FINDINGS; no subprocess, eval/exec, pickle or trust_remote_code in the reviewed server closure; no branch on item text, task id or benchmark name). The review findings were the runtime-download fallbacks in the author code (hf_hub_download for the calibration file and from_pretrained base resolution when the checkpoint is a Hub id); they were never reached: the merged checkpoint was loaded from the local pinned /model with its own /model/calibration.json, Hub offline, container --network none, every credential variable empty; the owner cleared the code for one offline measurement on those terms. Author settings kept: order-debiased scoring (JEZ_DEBIAS=1, given and reverse order averaged), JEZ_MAX_BATCH=16, the checkpoint's calibration temperatures unchanged; no score fitting, no prompt transformation. FIRST MEASUREMENT (found by our own intake of new public Hugging Face decision models; not a free-queue submission, not in the Decision Index at pick-up). CONTEXT CAP: the author server refuses a question whose state+question exceeds 32,000 tokens and a request above 64,000 tokens (HTTP 400 max_tokens_exceeded); our reviewed loopback status proxy maps exactly that answer to 422 and passes everything else through unchanged; on this draw that is exactly the 15 items of 76k-81k tokens, pre-registered by id and position; they count as wrong, like elf-1-12b, seb-9b and the NeoHorse rows on this draw. SAME FAMILY: jez-large-1.0.0 (Gemma-4-E4B base) and jez-small-1.0.0 (Qwen3.5-0.8B base) are two checkpoints of one author measured back to back on one VM. PROVENANCE, author statements (unverified): full fine-tune of Qwen/Qwen3.5-0.8B (card); trained on 278,999 gold-label examples from 20 public sources (reports/mix-v2-manifest.json, not audited by us), no teacher model; the card reports its own acceptance gate 2 (Tier 1 vs base) as not met and states no JevBench or Decision Index result. No rule against public items was found in the reviewed code. LATENCY: maximum single decision 0.054 s on this card; no decision came near the 10 s limit. Determinism probe on the first 50 positions, two passes: 50/50 bit-identical.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.2727272727272727,
       "cc": -51.42292222222223,
       "cc_by_type": {
        "choice": 8.333333333333334,
        "noul": -100,
        "score": -62.602100000000014
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.2376237623762376,
       "cc": -29.50564554060622,
       "cc_by_type": {
        "choice": 1.378579003181342,
        "noul": -92,
        "score": 2.104484375000004
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.25742574257425743,
       "cc": -26.079708841173883,
       "cc_by_type": {
        "choice": 8.94308943089431,
        "noul": -85.18518518518519,
        "score": -1.9970307692307676
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.3125,
       "cc": -14.807080158730153,
       "cc_by_type": {
        "choice": 42.85714285714286,
        "noul": -100,
        "score": 12.72161666666667
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.18888888888888888,
       "cc": -30.711684987450216,
       "cc_by_type": {
        "choice": -3.9461020211741884,
        "noul": -100,
        "score": 11.811047058823531
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.3514376996805112,
       "cc": -7.7625489739989595,
       "cc_by_type": {
        "choice": 28.678348155322713,
        "noul": -46.391752577319586,
        "score": -5.574242500000004
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2222222222222222,
       "cc": -37.92644264441173,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100,
        "score": -39.552523809523834
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.2581818181818182,
       "cc": -31.032599796681282,
       "cc_by_type": {
        "choice": 14.900554287577448,
        "noul": -88.88888888888889,
        "score": -19.10946478873241
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.19767441860465115,
       "cc": -37.16371999999999,
       "cc_by_type": {
        "choice": 0,
        "noul": -100,
        "score": -11.491159999999967
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.3541666666666667,
       "cc": 17.443408788282294,
       "cc_by_type": {
        "choice": 17.443408788282294
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 17.443408788282294
      },
      "perturb_pair": {
       "accuracy": 0.28125,
       "cc": -36.272838284438265,
       "cc_by_type": {
        "choice": 15.91591591591592,
        "noul": -80,
        "score": -44.73443076923071
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.23636363636363636,
       "cc": -31.477674975163378,
       "cc_by_type": {
        "choice": 14.215686274509835,
        "noul": -100,
        "score": -8.648711199999969
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -18.813943345543347,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": -50,
        "score": -30.25135384615385
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.14285714285714285,
       "cc": -52.631578947368425,
       "cc_by_type": {
        "choice": -5.263157894736845,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.23076923076923078,
       "cc": -46.459182918586784,
       "cc_by_type": {
        "choice": 17.050691244239633,
        "noul": -100,
        "score": -56.428239999999995
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.34375,
       "cc": -11.632148787995382,
       "cc_by_type": {
        "choice": 28.318584070796458,
        "noul": -75,
        "score": 11.784969565217395
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 0.30532675356921324,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 1.1022000000000087
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0.30532675356921324
      },
      "en": {
       "accuracy": 0.2545612510860122,
       "cc": -25.771539752756567,
       "cc_by_type": {
        "choice": 11.692559280457841,
        "noul": -84.89208633093526,
        "score": -4.1150922077922925
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.41935483870967744,
       "cc": -5.414871733358922,
       "cc_by_type": {
        "choice": 31.484502446982063,
        "noul": -25,
        "score": -22.72911764705883
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -24.175535883424413,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -33.333333333333336,
        "score": -8.045733333333338
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.35714285714285715,
       "cc": -4.599143962848291,
       "cc_by_type": {
        "choice": -5.882352941176468,
        "noul": 0,
        "score": -7.915078947368404
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.14285714285714285,
       "cc": -63.525431727188085,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100,
        "score": -106.77741250000001
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0,
       "cc": -66.66666666666667,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.5333333333333333,
       "cc": 3.8031803300795417,
       "cc_by_type": {
        "choice": 57.911145752143405,
        "noul": -33.333333333333336,
        "score": -13.168271428571444
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 3.8031803300795417
      },
      "ja": {
       "accuracy": 0.2,
       "cc": -36.35878715029565,
       "cc_by_type": {
        "choice": -0.07942811755361648,
        "noul": -80,
        "score": -28.99693333333333
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.1,
       "cc": -36.63425582010583,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": -21.013878571428602
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.2857142857142857,
       "cc": -23.8965,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -100,
        "score": -5.022833333333332
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.1,
       "cc": -48.03832796536796,
       "cc_by_type": {
        "choice": -3.8961038961038925,
        "noul": -100,
        "score": -40.21888
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.38461538461538464,
       "cc": -15.141638561872801,
       "cc_by_type": {
        "choice": 46.67795175624207,
        "noul": -80,
        "score": -12.102867441860464
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.32142857142857145,
       "cc": -19.605953278714505,
       "cc_by_type": {
        "choice": 15.144372421921032,
        "noul": -60,
        "score": -13.962232258064544
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": -10.591621631205669,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": -100,
        "score": -6.242949999999992
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.07692307692307693,
       "cc": -67.57375253623188,
       "cc_by_type": {
        "choice": -30.43478260869565,
        "noul": -100,
        "score": -72.28647500000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.2857142857142857,
       "cc": -33.33869301075269,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -35.49995000000001
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.3,
       "cc": -20.91184952919021,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": -33.333333333333336,
        "score": -27.70730000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.3333333333333333,
       "cc": 13.890430356493624,
       "cc_by_type": {
        "choice": 13.890430356493624
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 13.890430356493624
      },
      "noul": {
       "accuracy": 0.09066666666666667,
       "cc": -81.86666666666666,
       "cc_by_type": {
        "noul": -81.86666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.2986666666666667,
       "cc": -7.835111191335686,
       "cc_by_type": {
        "score": -7.835111191335686
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.09090909090909091,
       "cc": -47.05043333333334,
       "cc_by_type": {
        "choice": -12,
        "noul": -100,
        "score": -29.151300000000013
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.20833333333333334,
       "cc": -29.85727498009237,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -100,
        "score": 3.595876923076935
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.35714285714285715,
       "cc": 0.6096623306233061,
       "cc_by_type": {
        "choice": 18.699186991869915,
        "noul": -100,
        "score": 83.1298
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0.6096623306233061
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -39.28273179824562,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": -12.585037500000018
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": -34.78189145531402,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": -100,
        "score": -31.881906250000025
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.15789473684210525,
       "cc": -97.02319360587,
       "cc_by_type": {
        "choice": -0.62893081761006,
        "noul": -100,
        "score": -190.44064999999998
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": -11.012038037779222,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -71.42857142857143,
        "score": -22.391856410256427
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.18181818181818182,
       "cc": -38.69361545220422,
       "cc_by_type": {
        "choice": 5.9233449477351945,
        "noul": -100,
        "score": -22.004191304347852
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0,
       "cc": -54.962612962962965,
       "cc_by_type": {
        "choice": -25,
        "noul": -100,
        "score": -39.88783888888889
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -33.5989124813445,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": -100,
        "score": -10.860977272727279
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.12,
       "cc": -40.80497325581394,
       "cc_by_type": {
        "choice": -16.279069767441868,
        "noul": -100,
        "score": -6.135849999999965
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -28.290138844086027,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -20.354287500000012
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.37037037037037035,
       "cc": -16.352233594583083,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": -2.9914416666667165
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.42857142857142855,
       "cc": 5.444065174062047,
       "cc_by_type": {
        "choice": 40.389610389610404,
        "noul": -24.242424242424242,
        "score": 0.18500937499997816
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 5.444065174062047
      },
      "moderation_trust_safety": {
       "accuracy": 0.2222222222222222,
       "cc": -30.581622720258935,
       "cc_by_type": {
        "choice": 19.385796545105567,
        "noul": -100,
        "score": -11.130664705882376
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.3333333333333333,
       "cc": -29.952590382829786,
       "cc_by_type": {
        "choice": 16.434540389972145,
        "noul": -100,
        "score": -6.292311538461504
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -17.983279658070124,
       "cc_by_type": {
        "choice": 11.330049261083742,
        "noul": -50,
        "score": -15.279888235294115
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.17391304347826086,
       "cc": -39.641834009661835,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -100,
        "score": -17.476226666666683
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.25,
       "cc": -20.286962290066814,
       "cc_by_type": {
        "choice": 35.94306049822064,
        "noul": -100,
        "score": 3.1960526315789273
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.22516925019362,
      84.80314704701615
     ],
     "I": [
      2.6995341060300126,
      6.637224766650235
     ],
     "capability": [
      39.978396693489344,
      45.273986258801244
     ],
     "composite": null,
     "seed": 40
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.90437649433692,
     "capability": 43.23230567175719,
     "intelligence": 4.560234849177464
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3333333333333333,
      "calibration": 80.72034032786885,
      "cc": 13.890430356493624,
      "cc_by_type": {
       "choice": 13.890430356493624
      },
      "cc_tiered": 13.680704547532393,
      "cc_tiered_open": 13.819134266979594,
      "cc_tiered_sealed": 13.54227482808519,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.890430356493624
     },
     "noul": {
      "accuracy": 0.09066666666666667,
      "calibration": 85.37549597855228,
      "cc": -81.86666666666666,
      "cc_by_type": {
       "noul": -81.86666666666666
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2986666666666667,
      "calibration": 79.61729317658961,
      "cc": -7.835111191335686,
      "cc_by_type": {
       "score": -7.835111191335686
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 79.01899649379038,
     "capability": 41.76596099097298,
     "intelligence": 4.512925488155576
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.1032489115033149,
    "Emphasis on Accuracy 60:20:20": 0.06113964331995923,
    "Emphasis on Cost 20:20:60": 0.15701084363596937,
    "Emphasis on Speed 20:60:20": 0.15795268973544654,
    "Intelligence only": 0.03793338673363642
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "eljiwo / jiwidi (fhjaime96@gmail.com)",
   "axes": {
    "calibration": 78.54210067220151,
    "cost": 84.96035415353933,
    "intelligence": 15.872104931583502,
    "speed": 94.31864739858605
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14318770225758568,
      "ece_score": 71.36245954848286,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 71.36245954848286,
      "score_before_time_limit": 71.36245954848286,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.6428571428571428,
      "brier": 0.22635226125046812,
      "ece": 0.11375593371020894,
      "n": 364,
      "score": 77.24881325795822,
      "score_before_time_limit": 77.24881325795822,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9646739130434783,
      "interval90_width": 0.731884057971014,
      "n": 368,
      "nrps": 0.13949242305496334,
      "score": 87.01502921016349,
      "score_before_time_limit": 87.01502921016349,
      "time_limit_failed_share": 0,
      "top_ece": 0.060103496370883344
     }
    },
    "score": 78.54210067220151
   },
   "capability": 47.20710280189251,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.8B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 786,993 input + 0 output tokens over 1,500 decisions = 524.662 tokens/decision. GPU-time cost USD 0.00317191490873016/1,000; old list-price cost USD 0.00524662/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00317191490873016/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0031719149087301593
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jiwo 0.8b (Qwen3.5-0.8B backbone with a trained decision head, one forward pass, native probabilities, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "The author's own install line from their model card (pip install 'jiwo[cuda] @ git+https://github.com/jiwidi/jiwo@8716abab') and their own `jiwo serve`, nothing patched. The server reads JIWO_CHECKPOINT and calls from_pretrained with NO revision and there is no JIWO_REVISION setting, so it was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY and the served revision cannot have drifted to main. JIWO_MAX_LENGTH=65536, the author's own documented value for their published runs: the 8,192 default refuses longer prompts with HTTP 400 and never truncates. The trained context is 8,192 tokens, so the longest items of this draw are answered outside the range the model saw in training. CUDA graphs are OFF (JIWO_CUDA_GRAPHS unset): the author's own card says graphs agree with eager on only 99.7 % of fields, so eager is the exact path and the author's 14.5 ms headline uses a mode this measurement does not use. Requests are serialised by the author's own asyncio.Lock. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 20.53318982032003,
    "I_sealed": 11.211020042846975,
    "base": 15.872104931583502,
    "excess": -0.7510748038666595,
    "gap": 9.322169777473054,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 35.509112308155075,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 41.62162162162163,
       "hard": 30.16671909405474,
       "judge": 32.76203032300594,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.28571428571429,
       "hard": -39.13043478260869,
       "judge": -30.000000000000004,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 26.09045715280502,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.4799625425419,
       "hard": 18.20775252838498,
       "judge": 31.25102514541178,
       "standard": 21.920261717866516
      }
     },
     "sealed|choice": {
      "cc": 24.483755785527055,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 32.4225675252894,
       "hard": 18.37968561064086,
       "judge": 25.117954441682315,
       "standard": 31.771192281185392
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": -17.948717948717952,
       "hard": -69.6969696969697,
       "judge": -33.333333333333336,
       "standard": -54.385964912280706
      }
     },
     "sealed|score": {
      "cc": 9.149304343013874,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 26.922451761233834,
       "hard": 2.576868911219099,
       "judge": 8.09382448748016,
       "standard": 14.99082128079402
      }
     }
    }
   },
   "jevbench_score": 4.108504374494453,
   "key": "jiwo-0.8b",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (weights, model card); MIT (code, github.com/jiwidi/jiwo)",
   "listing": "ranked",
   "measured_in": "v1.6.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "eljiwo/jiwo-0.8b@52d217f8d68114e853b31ca364f8cb7da1b3c64d",
   "noul_decisive": {
    "acc_among_decisive": 0.7615384615384615,
    "decisive": 130,
    "decisive_rate": 0.35714285714285715,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.54210067220151,
     "cost": 78.40361168685378,
     "intelligence": 15.872104931583502,
     "speed": 94.33479301471377
    },
    "board_release": "v1.6.43",
    "capability": 47.20710280189251,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 98,
    "capability_rank": 125,
    "composite_ci95": {
     "A": [
      2.071813842220017,
      6.329337632277144
     ],
     "B": [
      1.533726731769256,
      4.973696434425022
     ],
     "C": [
      1.4387596126527897,
      4.395373355748017
     ]
    },
    "jevbench_score": 4.067767731552672,
    "measured_in": "v1.6.9",
    "method": "v1.6",
    "open_board_rank": 145,
    "rank": 185,
    "ranks": {
     "A": 185,
     "B": 187,
     "C": 186,
     "capability": 184
    },
    "scores": {
     "A": 4.067767731552672,
     "B": 3.1083596840941525,
     "C": 2.8248387024671335
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.18979715087916701,
     "p50_s_raw": 0.01989857543958351,
     "p95_s_adjusted": 0.19419185474980621,
     "p95_s_raw": 0.022095927374903113
    },
    "views": {
     "Balanced 33:33:33": 3.5006149102123802,
     "Emphasis on Accuracy 60:20:20": 2.372543895109552,
     "Emphasis on Cost 20:20:60": 4.503957993600416,
     "Emphasis on Speed 20:60:20": 4.684348850018089,
     "Intelligence only": 1.599423855434685
    }
   },
   "rank": 230,
   "ranked": true,
   "ranks": {
    "A": 230,
    "B": 231,
    "C": 230,
    "capability": 223
   },
   "repo": "https://huggingface.co/eljiwo/jiwo-0.8b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 4.108504374494453,
    "B": 3.1273153158005744,
    "C": 2.8531280378433705
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.1898998890304938,
    "p50_s_raw": 0.019949944515246898,
    "p95_s_adjusted": 0.19480968727730213,
    "p95_s_raw": 0.022404843638651074
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5172413793103449,
       "cc": 24.45957255405951,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 0,
        "score": 6.712050995511854
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 24.45957255405951
      },
      "calibration_gen": {
       "accuracy": 0.35714285714285715,
       "cc": -9.724218679588965,
       "cc_by_type": {
        "choice": 25.46583850931677,
        "noul": -61.53846153846154,
        "score": 6.89996699037787
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.37755102040816324,
       "cc": -5.9291709848934415,
       "cc_by_type": {
        "choice": 40.74074074074074,
        "noul": -78.57142857142857,
        "score": 20.04317487600751
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.7777777777777778,
       "cc": 71.6287638691846,
       "cc_by_type": {
        "choice": 64.70588235294117,
        "noul": 100,
        "score": 50.1804092546126
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 71.6287638691846
      },
      "jaggedness": {
       "accuracy": 0.24324324324324326,
       "cc": -31.490478394830276,
       "cc_by_type": {
        "choice": -0.04631773969430275,
        "noul": -94.87179487179486,
        "score": 0.446677426998332
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.48863636363636365,
       "cc": 31.193268142476445,
       "cc_by_type": {
        "choice": 28.471899674872283,
        "noul": 12.328767123287676,
        "score": 52.77913762926938
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 31.193268142476445
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.16129032258064516,
       "cc": -35.935636537648094,
       "cc_by_type": {
        "choice": -1.5384615384615408,
        "noul": -100,
        "score": -6.268448074482746
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.37142857142857144,
       "cc": -0.3712463104306612,
       "cc_by_type": {
        "choice": 29.243170622480974,
        "noul": -47.12643678160919,
        "score": 16.769527227836235
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.35714285714285715,
       "cc": -7.457195719833385,
       "cc_by_type": {
        "choice": 14.285714285714286,
        "noul": -40,
        "score": 3.3426985547855614
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.49122807017543857,
       "cc": 35.62708102108768,
       "cc_by_type": {
        "choice": 35.62708102108768
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 35.62708102108768
      },
      "perturb_pair": {
       "accuracy": 0.6,
       "cc": 31.852412487051,
       "cc_by_type": {
        "choice": 52.81782437745741,
        "noul": -14.28571428571429,
        "score": 57.02512736940989
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 31.852412487051
      },
      "usecase": {
       "accuracy": 0.29957805907172996,
       "cc": -18.97090524037614,
       "cc_by_type": {
        "choice": 29.46617845442283,
        "noul": -82.45614035087719,
        "score": -3.9227538246740634
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.45454545454545453,
       "cc": -0.15643741530528152,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": -50,
        "score": 29.530687754084152
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5,
       "cc": -12.711864406779661,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.2916666666666667,
       "cc": 0.2625190049768242,
       "cc_by_type": {
        "choice": 5.325443786982243,
        "noul": -19.999999999999996,
        "score": 15.462113227948226
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0.2625190049768242
      },
      "de": {
       "accuracy": 0.4222222222222222,
       "cc": 6.827684779744186,
       "cc_by_type": {
        "choice": 29.390464695232346,
        "noul": -11.111111111111116,
        "score": 2.2037007551113263
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 6.827684779744186
      },
      "el": {
       "accuracy": 0.16666666666666666,
       "cc": -54.072622649773365,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -100,
        "score": -65.83232578064539
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.3735478105451296,
       "cc": -4.109282546744619,
       "cc_by_type": {
        "choice": 25.29444756029158,
        "noul": -49.62406015037594,
        "score": 12.001764949850502
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.6071428571428571,
       "cc": 34.262221513293575,
       "cc_by_type": {
        "choice": 86.90909090909093,
        "noul": 9.090909090909083,
        "score": 6.786664539880705
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 34.262221513293575
      },
      "fi": {
       "accuracy": 0.3333333333333333,
       "cc": -31.690140845070424,
       "cc_by_type": {
        "choice": 36.61971830985915,
        "noul": -100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.4,
       "cc": -5.790035284708807,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": -100,
        "score": 33.405634625422245
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.06666666666666667,
       "cc": -36.64150500544096,
       "cc_by_type": {
        "choice": -24.22360248447205,
        "noul": -100,
        "score": 14.299087468149175
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.3333333333333333,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7647058823529411,
       "cc": 55.395640882380825,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 50,
        "score": 66.96266312669114
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 55.395640882380825
      },
      "ja": {
       "accuracy": 0.21875,
       "cc": -28.34012514670054,
       "cc_by_type": {
        "choice": 10.569105691056912,
        "noul": -100,
        "score": 4.410518868841473
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.2727272727272727,
       "cc": -19.50727275296486,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": 7.0519522329086985
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.6363636363636364,
       "cc": 40.89245662958265,
       "cc_by_type": {
        "choice": 78.02197802197803,
        "noul": 0,
        "score": 44.655391866769904
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 40.89245662958265
      },
      "nl": {
       "accuracy": 0.23076923076923078,
       "cc": -23.738139726914895,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": -33.333333333333336,
        "score": -37.60253431537792
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.14285714285714285,
       "cc": -49.18032786885246,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.45,
       "cc": 25.233491597647614,
       "cc_by_type": {
        "choice": 60.64713598600787,
        "noul": 0,
        "score": 15.05333880693498
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 25.233491597647614
      },
      "pt": {
       "accuracy": 0.7222222222222222,
       "cc": 59.348158787470005,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 44.711143029076695
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 59.348158787470005
      },
      "sv": {
       "accuracy": 0,
       "cc": -47.36188981647681,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": -8.752336116097092
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.45454545454545453,
       "cc": 0.2881809678820299,
       "cc_by_type": {
        "choice": 21.052631578947373,
        "noul": -33.333333333333336,
        "score": 13.145244658032052
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0.2881809678820299
      },
      "uk": {
       "accuracy": 0.45454545454545453,
       "cc": -12.288605896536685,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": 50.09070404952038
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.375,
       "cc": 10.800875659098933,
       "cc_by_type": {
        "choice": 2.702702702702704,
        "noul": 33.33333333333333,
        "score": -3.6334090587392343
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 10.800875659098933
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.43333333333333335,
       "cc": 26.58490240110555,
       "cc_by_type": {
        "choice": 26.58490240110555
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 26.58490240110555
      },
      "noul": {
       "accuracy": 0.264,
       "cc": -47.199999999999996,
       "cc_by_type": {
        "noul": -47.199999999999996
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.37333333333333335,
       "cc": 13.164164091005937,
       "cc_by_type": {
        "score": 13.164164091005937
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 13.164164091005937
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.375,
       "cc": -24.495021091479135,
       "cc_by_type": {
        "choice": 30.807248764415156,
        "noul": -100,
        "score": -4.292312038852564
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": -33.15167050235345,
       "cc_by_type": {
        "choice": 21.440261865793783,
        "noul": -100,
        "score": -20.895273372854128
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.3,
       "cc": -1.166880782233278,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -75,
        "score": 36.01548668555823
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.2692307692307692,
       "cc": -29.51868765932299,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -100,
        "score": -4.218713580378597
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2777777777777778,
       "cc": -19.515561208758683,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100,
        "score": -2.6460625082636247
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.2413793103448276,
       "cc": -22.650212849237985,
       "cc_by_type": {
        "choice": 5.9829059829059785,
        "noul": -60,
        "score": -13.933544530619923
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.34615384615384615,
       "cc": -8.626664926886974,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -75,
        "score": 14.337396523686897
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.20689655172413793,
       "cc": -43.06325104574836,
       "cc_by_type": {
        "choice": 4.950495049504946,
        "noul": -100,
        "score": -34.140248186750014
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.42857142857142855,
       "cc": 16.29704619319791,
       "cc_by_type": {
        "choice": 52.38095238095238,
        "noul": -100,
        "score": 96.51018619864135
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 16.29704619319791
      },
      "knowledge_graphs": {
       "accuracy": 0.13043478260869565,
       "cc": -36.43239114303143,
       "cc_by_type": {
        "choice": -0.7633587786259586,
        "noul": -100,
        "score": -8.53381465046834
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.24,
       "cc": -15.40910199821853,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": -75,
        "score": -6.186655588151524
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.38461538461538464,
       "cc": 48.23720641132996,
       "cc_by_type": {
        "choice": 18.032786885245905,
        "noul": 100,
        "score": 26.678832348743953
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 48.23720641132996
      },
      "llm_guardrails": {
       "accuracy": 0.3103448275862069,
       "cc": -8.538148364008457,
       "cc_by_type": {
        "choice": 44.95412844036697,
        "noul": -75,
        "score": 4.431426467607658
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.5133333333333333,
       "cc": 32.82106671729687,
       "cc_by_type": {
        "choice": 42.106673813990895,
        "noul": 9.090909090909083,
        "score": 47.265617246990644
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 32.82106671729687
      },
      "moderation_trust_safety": {
       "accuracy": 0.1875,
       "cc": -18.741819385691944,
       "cc_by_type": {
        "choice": -10.526315789473685,
        "noul": -100,
        "score": 54.30085763239786
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.23809523809523808,
       "cc": -18.91047217979512,
       "cc_by_type": {
        "choice": 9.547738693467341,
        "noul": -100,
        "score": 33.7208447671473
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.46153846153846156,
       "cc": 6.905355691048825,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": 33.33333333333333,
        "score": -56.0134926752812
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 6.905355691048825
      },
      "scientific_discovery": {
       "accuracy": 0.25,
       "cc": -23.30406902667455,
       "cc_by_type": {
        "choice": 20.31872509960159,
        "noul": -100,
        "score": 9.769067820374755
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.2727272727272727,
       "cc": -24.90976631788278,
       "cc_by_type": {
        "choice": 28.40095465393794,
        "noul": -100,
        "score": -3.1302536075862752
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.49837033035136,
      80.76387849103644
     ],
     "I": [
      12.343340329004636,
      18.854234371958732
     ],
     "capability": [
      43.298750810210294,
      49.02128383708443
     ],
     "composite": null,
     "seed": 22
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 78.54210067220151,
     "capability": 47.20710280189251,
     "intelligence": 15.872104931583502
    },
    "per_type": {
     "choice": {
      "accuracy": 0.43333333333333335,
      "calibration": 71.36245954848286,
      "cc": 26.58490240110555,
      "cc_by_type": {
       "choice": 26.58490240110555
      },
      "cc_tiered": 29.996434046841067,
      "cc_tiered_open": 35.509112308155075,
      "cc_tiered_sealed": 24.483755785527055,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 26.58490240110555
     },
     "noul": {
      "accuracy": 0.264,
      "calibration": 77.24881325795822,
      "cc": -47.199999999999996,
      "cc_by_type": {
       "noul": -47.199999999999996
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.37333333333333335,
      "calibration": 87.01502921016349,
      "cc": 13.164164091005937,
      "cc_by_type": {
       "score": 13.164164091005937
      },
      "cc_tiered": 17.61988074790945,
      "cc_tiered_open": 26.09045715280502,
      "cc_tiered_sealed": 9.149304343013874,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 13.164164091005937
     }
    },
    "raw_on_A": {
     "calibration": 73.78928446595333,
     "capability": 44.008100936829386,
     "intelligence": 14.22691740770544
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 3.5408997876347614,
    "Emphasis on Accuracy 60:20:20": 2.3835714123916016,
    "Emphasis on Cost 20:20:60": 4.625994692555262,
    "Emphasis on Speed 20:60:20": 4.72737149338453,
    "Intelligence only": 1.599423855434685
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "eljiwo / jiwidi (fhjaime96@gmail.com)",
   "axes": {
    "calibration": 85.66971124396188,
    "cost": 78.63190282478453,
    "intelligence": 34.784391844693545,
    "speed": 92.17972487720941
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0747562710407059,
      "ece_score": 85.04874579185882,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 85.04874579185882,
      "score_before_time_limit": 85.04874579185882,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.37362637362637363,
      "brier": 0.1479946142436308,
      "ece": 0.06661725466202273,
      "n": 364,
      "score": 86.67654906759546,
      "score_before_time_limit": 86.67654906759546,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9402173913043478,
      "interval90_width": 0.48007246376811585,
      "n": 368,
      "nrps": 0.1073762318235206,
      "score": 85.28383887243137,
      "score_before_time_limit": 85.28383887243137,
      "time_limit_failed_share": 0,
      "top_ece": 0.09347349536392596
     }
    },
    "score": 85.66971124396188
   },
   "capability": 60.227051544327715,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 786,993 input + 0 output tokens over 1,500 decisions = 524.662 tokens/decision. GPU-time cost USD 0.00515548949001245/1,000; old list-price cost USD 0.01573986/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00515548949001245/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005155489490012448
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jiwo 4b (Qwen3.5-4B backbone with a trained decision head, one forward pass, native probabilities, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "The author's own install line from their model card (pip install 'jiwo[cuda] @ git+https://github.com/jiwidi/jiwo@8716abab') and their own `jiwo serve`, nothing patched. The server reads JIWO_CHECKPOINT and calls from_pretrained with NO revision and there is no JIWO_REVISION setting, so it was pointed at the LOCAL PINNED SNAPSHOT DIRECTORY and the served revision cannot have drifted to main. JIWO_MAX_LENGTH=65536, the author's own documented value for their published runs: the 8,192 default refuses longer prompts with HTTP 400 and never truncates. The trained context is 8,192 tokens, so the longest items of this draw are answered outside the range the model saw in training. CUDA graphs are OFF (JIWO_CUDA_GRAPHS unset): the author's own card says graphs agree with eager on only 99.7 % of fields, so eager is the exact path and the author's 14.5 ms headline uses a mode this measurement does not use. Requests are serialised by the author's own asyncio.Lock. evaluator-owned Lium RTX 5090 32 GB pod (pod calm-wolf-31, USD 0.72/h; Brasov, Romania, EU by the RIPE whois of its public IP, receipt r90/receipts/POD-IP-WHOIS-R90.txt); pod-level iptables egress cut installed, verified and TIMESTAMPED pod-side (logs/EGRESS-CUT-R90.log) BEFORE the sealed input was uploaded, so the ordering is provable from pod evidence and not only from the session transcript; HF/transformers offline, no credentials in the environment, one decision per request, no batching, no retry. The five rows ran strictly one at a time on the one card, each server started with setsid and stopped by its exact process group with a card-free check in between.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.202737997549946,
    "I_sealed": 28.366045691837137,
    "base": 34.784391844693545,
    "excess": 2.763447724373096,
    "gap": 12.836692305712809,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 61.6558109869045,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864866,
       "hard": 58.47750865051904,
       "judge": 60.44825313117996,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 26.298826777087655,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 21.739130434782616,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 35.653576228657684,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.10967244172717,
       "hard": 38.322559765790174,
       "judge": 32.507251953307225,
       "standard": 27.80704746088366
      }
     },
     "sealed|choice": {
      "cc": 51.626839698704124,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 64.33413286056941,
       "hard": 48.059799934044186,
       "judge": 51.60062909035564,
       "standard": 52.446588559614064
      }
     },
     "sealed|noul": {
      "cc": 4.118819776714509,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 33.33333333333333,
       "hard": -7.57575757575758,
       "judge": 13.888888888888884,
       "standard": -1.7543859649122862
      }
     },
     "sealed|score": {
      "cc": 29.352477600092772,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 53.37012049696441,
       "hard": 14.78666626313042,
       "judge": 38.03240184856901,
       "standard": 33.455392452867294
      }
     }
    }
   },
   "jevbench_score": 30.25493665442467,
   "key": "jiwo-4b",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (weights, model card); MIT (code, github.com/jiwidi/jiwo)",
   "listing": "ranked",
   "measured_in": "v1.6.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "eljiwo/jiwo-4b@9adc559ff7ec5baffade3998f7bca85da0d124ed",
   "noul_decisive": {
    "acc_among_decisive": 0.8991228070175439,
    "decisive": 228,
    "decisive_rate": 0.6263736263736264,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.66971124396188,
     "cost": 64.0899740452639,
     "intelligence": 34.784391844693545,
     "speed": 92.44547609658223
    },
    "board_release": "v1.6.43",
    "capability": 60.227051544327715,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 60,
    "capability_rank": 85,
    "composite_ci95": {
     "A": [
      18.62937312636143,
      39.08706741447612
     ],
     "B": [
      15.795090114565683,
      34.93479875570867
     ],
     "C": [
      12.937064671084322,
      27.143796815608415
     ]
    },
    "jevbench_score": 28.962932355698086,
    "measured_in": "v1.6.9",
    "method": "v1.6",
    "open_board_rank": 71,
    "rank": 103,
    "ranks": {
     "A": 103,
     "B": 104,
     "C": 113,
     "capability": 131
    },
    "scores": {
     "A": 28.962932355698086,
     "B": 25.315480694857687,
     "C": 20.113147469234782
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21877912301570177,
     "p50_s_raw": 0.034389561507850885,
     "p95_s_adjusted": 0.26028342521749437,
     "p95_s_raw": 0.05514171260874719
    },
    "views": {
     "Balanced 33:33:33": 26.318232161775516,
     "Emphasis on Accuracy 60:20:20": 21.478631369812923,
     "Emphasis on Cost 20:20:60": 28.0163293235414,
     "Emphasis on Speed 20:60:20": 31.50792357388672,
     "Intelligence only": 16.835004451338282
    }
   },
   "rank": 166,
   "ranked": true,
   "ranks": {
    "A": 166,
    "B": 167,
    "C": 168,
    "capability": 164
   },
   "repo": "https://huggingface.co/eljiwo/jiwo-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 30.25493665442467,
    "B": 26.094688812120623,
    "C": 21.010372676683797
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.222632587980479,
    "p50_s_raw": 0.0363162939902395,
    "p95_s_adjusted": 0.27191851355601104,
    "p95_s_raw": 0.06095925677800551
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7586206896551724,
       "cc": 63.41069040078068,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 66.66666666666667,
        "score": 56.8987378690087
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 63.41069040078068
      },
      "calibration_gen": {
       "accuracy": 0.5446428571428571,
       "cc": 21.306873408130947,
       "cc_by_type": {
        "choice": 51.08695652173913,
        "noul": 0,
        "score": 12.833663702653709
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 21.306873408130947
      },
      "composite_gen": {
       "accuracy": 0.6020408163265306,
       "cc": 40.276766724193266,
       "cc_by_type": {
        "choice": 62.96296296296296,
        "noul": 21.42857142857142,
        "score": 36.438765781045426
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 40.276766724193266
      },
      "fairness": {
       "accuracy": 1,
       "cc": 97.09800473660214,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 91.29401420980636
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 97.09800473660214
      },
      "jaggedness": {
       "accuracy": 0.372972972972973,
       "cc": -11.082947542546755,
       "cc_by_type": {
        "choice": 29.133858267716537,
        "noul": -69.23076923076923,
        "score": 6.848068335412427
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.6893939393939394,
       "cc": 58.2269224408714,
       "cc_by_type": {
        "choice": 51.69530887134232,
        "noul": 50.68493150684932,
        "score": 72.30052694442254
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 58.2269224408714
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3870967741935484,
       "cc": -16.440841671706764,
       "cc_by_type": {
        "choice": 16.923076923076923,
        "noul": -75,
        "score": 8.754398061802782
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6158730158730159,
       "cc": 34.72665534729893,
       "cc_by_type": {
        "choice": 57.00850873264667,
        "noul": 17.24137931034482,
        "score": 29.930077998905315
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 34.72665534729893
      },
      "negation_pair_gen": {
       "accuracy": 0.5892857142857143,
       "cc": 2.0857022253018394,
       "cc_by_type": {
        "choice": 57.142857142857146,
        "noul": 10.000000000000009,
        "score": -60.885750466951635
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 2.0857022253018394
      },
      "option_order": {
       "accuracy": 0.9122807017543859,
       "cc": 88.90122086570477,
       "cc_by_type": {
        "choice": 88.90122086570477
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 88.90122086570477
      },
      "perturb_pair": {
       "accuracy": 0.85,
       "cc": 73.88280168114402,
       "cc_by_type": {
        "choice": 84.27260812581913,
        "noul": 57.14285714285714,
        "score": 80.23293977475579
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 73.88280168114402
      },
      "usecase": {
       "accuracy": 0.5485232067510548,
       "cc": 27.448440267073206,
       "cc_by_type": {
        "choice": 51.4357294276354,
        "noul": 1.7543859649122862,
        "score": 29.155205408671925
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 27.448440267073206
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.45454545454545453,
       "cc": -16.233054801263584,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": -50,
        "score": -18.699164403790757
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5,
       "cc": 28.531073446327685,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 28.531073446327685
      },
      "da": {
       "accuracy": 0.5833333333333334,
       "cc": 40.559189285398894,
       "cc_by_type": {
        "choice": 40.8284023668639,
        "noul": 19.999999999999996,
        "score": 60.84916548933277
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 40.559189285398894
      },
      "de": {
       "accuracy": 0.6888888888888889,
       "cc": 43.45157490330706,
       "cc_by_type": {
        "choice": 72.84248642124321,
        "noul": 33.33333333333333,
        "score": 24.178904955344628
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 43.45157490330706
      },
      "el": {
       "accuracy": 0.4444444444444444,
       "cc": -27.556284563855588,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": -60,
        "score": -62.42788983614507
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.5889186773905273,
       "cc": 30.320127791507446,
       "cc_by_type": {
        "choice": 53.7857543466068,
        "noul": 5.263157894736836,
        "score": 31.911471133178704
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 30.320127791507446
      },
      "es": {
       "accuracy": 0.75,
       "cc": 53.31121705049497,
       "cc_by_type": {
        "choice": 73.81818181818181,
        "noul": 45.45454545454546,
        "score": 40.660923878757615
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 53.31121705049497
      },
      "fi": {
       "accuracy": 0.4444444444444444,
       "cc": 24.4131455399061,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 24.4131455399061
      },
      "fr": {
       "accuracy": 0.6,
       "cc": 19.936766375030352,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": -50,
        "score": 35.19816936486537
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 19.936766375030352
      },
      "hi": {
       "accuracy": 0.43333333333333335,
       "cc": 12.211958226018675,
       "cc_by_type": {
        "choice": 25.465838509316765,
        "noul": 0,
        "score": 11.170036168739262
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 12.211958226018675
      },
      "id": {
       "accuracy": 0.8333333333333334,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 50
      },
      "it": {
       "accuracy": 0.8235294117647058,
       "cc": 65.35071606453869,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 50,
        "score": 46.05214819361604
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 65.35071606453869
      },
      "ja": {
       "accuracy": 0.4375,
       "cc": 12.03476070227424,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": -14.28571428571429,
        "score": 15.430646799041082
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 12.03476070227424
      },
      "ko": {
       "accuracy": 0.6363636363636364,
       "cc": 57.974720722270284,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": 6.711047412712501
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 57.974720722270284
      },
      "mixed": {
       "accuracy": 1,
       "cc": 89.40690554679048,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 68.22071664037144
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 89.40690554679048
      },
      "nl": {
       "accuracy": 0.5384615384615384,
       "cc": 30.176686204383895,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 0,
        "score": 40.66933437916839
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 30.176686204383895
      },
      "no": {
       "accuracy": 0.7142857142857143,
       "cc": 67.21311475409837,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 67.21311475409837
      },
      "pl": {
       "accuracy": 0.6,
       "cc": 33.93999941140274,
       "cc_by_type": {
        "choice": 76.38828159160472,
        "noul": 0,
        "score": 25.431716642603497
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 33.93999941140274
      },
      "pt": {
       "accuracy": 0.8888888888888888,
       "cc": 77.91153300960076,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": 66.66666666666667,
        "score": 85.15838462344217
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 77.91153300960076
      },
      "sv": {
       "accuracy": 0.375,
       "cc": -8.765750483399948,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": 50,
        "score": -42.96391811686651
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.6363636363636364,
       "cc": 24.254058362470982,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 33.33333333333333,
        "score": -34.25536877223618
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 24.254058362470982
      },
      "uk": {
       "accuracy": 0.45454545454545453,
       "cc": -18.250772299457207,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": 32.20420484075882
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.75,
       "cc": 61.21814460262698,
       "cc_by_type": {
        "choice": 67.56756756756756,
        "noul": 33.33333333333333,
        "score": 82.75353290698004
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 61.21814460262698
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6453333333333333,
       "cc": 54.05078597339782,
       "cc_by_type": {
        "choice": 54.05078597339782
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 54.05078597339782
      },
      "noul": {
       "accuracy": 0.5466666666666666,
       "cc": 9.333333333333327,
       "cc_by_type": {
        "noul": 9.333333333333327
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 9.333333333333327
      },
      "score": {
       "accuracy": 0.5413333333333333,
       "cc": 31.101047707694065,
       "cc_by_type": {
        "score": 31.101047707694065
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 31.101047707694065
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5833333333333334,
       "cc": 24.071004328629304,
       "cc_by_type": {
        "choice": 40.691927512355846,
        "noul": -19.999999999999996,
        "score": 51.52108547353207
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 24.071004328629304
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 39.264007955407884,
       "cc_by_type": {
        "choice": 41.08019639934533,
        "noul": 60.00000000000001,
        "score": 16.711827466878294
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 39.264007955407884
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 20.00867761558955,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -25,
        "score": 49.54216187902672
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 20.00867761558955
      },
      "demand_forecasting": {
       "accuracy": 0.38461538461538464,
       "cc": -10.818142384890914,
       "cc_by_type": {
        "choice": 27.710843373493972,
        "noul": -50,
        "score": -10.165270528166715
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5,
       "cc": 24.55248819664185,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -19.999999999999996,
        "score": 30.92454533526716
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 24.55248819664185
      },
      "feature_extraction": {
       "accuracy": 0.4482758620689655,
       "cc": 12.840294527779639,
       "cc_by_type": {
        "choice": 23.076923076923077,
        "noul": -19.999999999999996,
        "score": 35.44396050641584
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 12.840294527779639
      },
      "financial_crime": {
       "accuracy": 0.6153846153846154,
       "cc": 42.181818059125035,
       "cc_by_type": {
        "choice": 47.82608695652174,
        "noul": 75,
        "score": 3.7193672208533535
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 42.181818059125035
      },
      "gaming": {
       "accuracy": 0.5172413793103449,
       "cc": 15.436535058528824,
       "cc_by_type": {
        "choice": 52.475247524752476,
        "noul": -14.28571428571429,
        "score": 8.120071936548289
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 15.436535058528824
      },
      "insurance_claims": {
       "accuracy": 0.5,
       "cc": 2.555674036706544,
       "cc_by_type": {
        "choice": 68.25396825396825,
        "noul": -60,
        "score": -0.5869461438486212
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 2.555674036706544
      },
      "knowledge_graphs": {
       "accuracy": 0.4782608695652174,
       "cc": 14.125622525737157,
       "cc_by_type": {
        "choice": 35.87786259541985,
        "noul": -19.999999999999996,
        "score": 26.499004981791618
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 14.125622525737157
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 11.047439957865064,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -50,
        "score": -0.59751752477879
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 11.047439957865064
      },
      "legal_compliance": {
       "accuracy": 0.46153846153846156,
       "cc": -11.127349456353167,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": 32.19172212274377
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.6551724137931034,
       "cc": 53.749288065206805,
       "cc_by_type": {
        "choice": 63.30275229357799,
        "noul": 50,
        "score": 47.945111902042434
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 53.749288065206805
      },
      "model_routing": {
       "accuracy": 0.7133333333333334,
       "cc": 58.29415940370154,
       "cc_by_type": {
        "choice": 64.62074510855,
        "noul": 59.09090909090908,
        "score": 51.17082401164556
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 58.29415940370154
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 16.05980603069598,
       "cc_by_type": {
        "choice": 52.63157894736842,
        "noul": -60,
        "score": 55.547839144719525
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 16.05980603069598
      },
      "recruiting": {
       "accuracy": 0.6190476190476191,
       "cc": 33.863204600776136,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 0,
        "score": 31.740367571172644
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 33.863204600776136
      },
      "risk_assessment": {
       "accuracy": 0.3076923076923077,
       "cc": -16.123855817090945,
       "cc_by_type": {
        "choice": 5.660377358490565,
        "noul": -33.333333333333336,
        "score": -20.698611476430063
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.375,
       "cc": -7.830166742245569,
       "cc_by_type": {
        "choice": 28.286852589641434,
        "noul": -71.42857142857143,
        "score": 19.65121861219329
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.8181818181818182,
       "cc": 66.53707551936073,
       "cc_by_type": {
        "choice": 85.68019093078759,
        "noul": 66.66666666666667,
        "score": 47.26436896062795
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 66.53707551936073
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.08166368366025,
      86.75710341088941
     ],
     "I": [
      29.097201235740425,
      39.427338008588556
     ],
     "capability": [
      55.64492730946361,
      62.239450216352004
     ],
     "composite": null,
     "seed": 23
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.66971124396188,
     "capability": 60.227051544327715,
     "intelligence": 34.784391844693545
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6453333333333333,
      "calibration": 85.04874579185882,
      "cc": 54.05078597339782,
      "cc_by_type": {
       "choice": 54.05078597339782
      },
      "cc_tiered": 56.64132534280431,
      "cc_tiered_open": 61.6558109869045,
      "cc_tiered_sealed": 51.626839698704124,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.05078597339782
     },
     "noul": {
      "accuracy": 0.5466666666666666,
      "calibration": 86.67654906759546,
      "cc": 9.333333333333327,
      "cc_by_type": {
       "noul": 9.333333333333327
      },
      "cc_tiered": 15.208823276901082,
      "cc_tiered_open": 26.298826777087655,
      "cc_tiered_sealed": 4.118819776714509,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 9.333333333333327
     },
     "score": {
      "accuracy": 0.5413333333333333,
      "calibration": 85.28383887243137,
      "cc": 31.101047707694065,
      "cc_by_type": {
       "score": 31.101047707694065
      },
      "cc_tiered": 32.503026914375226,
      "cc_tiered_open": 35.653576228657684,
      "cc_tiered_sealed": 29.352477600092772,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 31.101047707694065
     }
    },
    "raw_on_A": {
     "calibration": 82.12950185196465,
     "capability": 58.61658974199523,
     "intelligence": 35.1036776320258
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 27.7542161494779,
    "Emphasis on Accuracy 60:20:20": 22.036938312021203,
    "Emphasis on Cost 20:20:60": 31.124475865466792,
    "Emphasis on Speed 20:60:20": 32.69654017322357,
    "Intelligence only": 16.835004451338282
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "kirp",
   "axes": {
    "calibration": 89.9313809717458,
    "cost": 72.4983280625774,
    "intelligence": 51.42208418374838,
    "speed": 90.38067871211373
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.074450619123307,
      "ece_score": 85.1098761753386,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 85.1098761753386,
      "score_before_time_limit": 85.1098761753386,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.25876010781671155,
      "brier": 0.09167755563129923,
      "ece": 0.04206042293296505,
      "n": 371,
      "score": 91.58791541340699,
      "score_before_time_limit": 91.58791541340699,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9539295392953929,
      "interval90_width": 0.3337850045167119,
      "n": 369,
      "nrps": 0.06863762470007446,
      "score": 93.09635132649183,
      "score_before_time_limit": 93.09635132649183,
      "time_limit_failed_share": 0,
      "top_ece": 0.03471767438504456
     }
    },
    "score": 89.9313809717458
   },
   "capability": 70.67673257774709,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 764,897 input + 1,477 output tokens over 1,477 decisions = 518.872037915 tokens/decision. GPU-time cost USD 0.0082551011283206/1,000; old list-price cost USD 0.163085011848341/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0082551011283206/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.160584375
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.008255101128320605
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JPT-35B-A3B (Qwen3.5-35B-A3B fine-tune)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 52.77555274931294,
    "I_sealed": 50.06861561818382,
    "base": 51.42208418374838,
    "excess": 0.13354948869087968,
    "gap": 2.7069371311291235,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 71.66771648987253,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 71.68921044353571,
       "judge": 72.31377719182596,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 33.899930986887505,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 13.043478260869556,
       "judge": 30.000000000000004,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 52.759010771178794,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 66.68236487691772,
       "hard": 47.969282366711376,
       "judge": 51.97130113741173,
       "standard": 56.55835497789477
      }
     },
     "sealed|choice": {
      "cc": 64.13898950109635,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 59.27601809954751,
       "judge": 64.61824953445065,
       "standard": 67.8531838896416
      }
     },
     "sealed|noul": {
      "cc": 31.905895053509607,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72,
       "hard": 14.563106796116498,
       "judge": 26.315789473684205,
       "standard": 54.92957746478873
      }
     },
     "sealed|score": {
      "cc": 54.16096229994551,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.36028549114205,
       "hard": 51.20129834773479,
       "judge": 55.62940093493124,
       "standard": 51.77797065629009
      }
     }
    }
   },
   "jevbench_score": 72.1706145001871,
   "key": "jpt-35b-a3b",
   "last_measured_on": "2026-10-06",
   "licence": "cc-by-nc-4.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "kirp/jpt-35b-a3b@f9b45a14df4f7eaf5551f1ab26607237604a8588",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9381818181818182,
    "decisive_rate": 0.7412398921832885,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.9313809717458,
     "cost": 33.627578514713974,
     "intelligence": 51.42208418374838,
     "speed": 90.51950606084544
    },
    "board_release": "v1.6.43",
    "capability": 70.67673257774709,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.22909286309626,
      25.862384694814303
     ],
     "B": [
      19.5180684428127,
      25.72730452057616
     ],
     "C": [
      14.047981154927955,
      22.379413062160616
     ]
    },
    "jevbench_score": 25.357657195103958,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 78,
    "rank": 111,
    "ranks": {
     "A": 111,
     "B": 107,
     "C": 121,
     "capability": 73
    },
    "scores": {
     "A": 25.357657195103958,
     "B": 24.9082855667434,
     "C": 18.62541574870352
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.29123902405845004,
     "p50_s_raw": 0.070619512029225,
     "p95_s_adjusted": 0.30464904906693846,
     "p95_s_raw": 0.07732452453346922
    },
    "views": {
     "Balanced 33:33:33": 22.529266440821832,
     "Emphasis on Accuracy 60:20:20": 22.81579765829606,
     "Emphasis on Cost 20:20:60": 18.893073743976853,
     "Emphasis on Speed 20:60:20": 27.47147293414375,
     "Intelligence only": 23.25952583784173
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 55,
   "ranked": true,
   "ranks": {
    "A": 55,
    "B": 83,
    "C": 96,
    "capability": 93
   },
   "repo": "https://huggingface.co/kirp/jpt-35b-a3b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 72.1706145001871,
    "B": 66.78142629424116,
    "C": 53.00993264334113
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2935272410162725,
    "p50_s_raw": 0.07176362050813623,
    "p95_s_adjusted": 0.3120927760901395,
    "p95_s_raw": 0.08104638804506976
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-35B-A3B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.28164810054457,
      90.58368457070935
     ],
     "I": [
      45.56151371729817,
      55.912127823531534
     ],
     "capability": [
      65.85556031844814,
      72.40411689862715
     ],
     "composite": null,
     "seed": 58
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.9313809717458,
     "capability": 70.67673257774709,
     "intelligence": 51.42208418374838
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7386666666666667,
      "calibration": 85.1098761753386,
      "cc": 66.57919915121775,
      "cc_by_type": {
       "choice": 66.57919915121775
      },
      "cc_tiered": 67.90335299548444,
      "cc_tiered_open": 71.66771648987253,
      "cc_tiered_sealed": 64.13898950109635,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 66.57919915121775
     },
     "noul": {
      "accuracy": 0.688,
      "calibration": 91.58791541340699,
      "cc": 37.59999999999999,
      "cc_by_type": {
       "noul": 37.59999999999999
      },
      "cc_tiered": 32.902913020198554,
      "cc_tiered_open": 33.899930986887505,
      "cc_tiered_sealed": 31.905895053509607,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 37.59999999999999
     },
     "score": {
      "accuracy": 0.6906666666666667,
      "calibration": 93.09635132649183,
      "cc": 54.13323074000276,
      "cc_by_type": {
       "score": 54.13323074000276
      },
      "cc_tiered": 53.45998653556215,
      "cc_tiered_open": 52.759010771178794,
      "cc_tiered_sealed": 54.16096229994551,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.13323074000276
     }
    },
    "raw_on_A": {
     "calibration": 89.37706553915206,
     "capability": 72.5236455910987,
     "intelligence": 55.67022564304535
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 67.71301218353693,
    "Emphasis on Accuracy 60:20:20": 60.09729099231004,
    "Emphasis on Cost 20:20:60": 69.54927664252405,
    "Emphasis on Speed 20:60:20": 75.26351173788636,
    "Intelligence only": 51.42208418374838
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "kirp",
   "axes": {
    "calibration": 86.88996927941005,
    "cost": 78.85487162848112,
    "intelligence": 40.7504160092812,
    "speed": 87.81344131912249
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06574074427929287,
      "ece_score": 86.85185114414142,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 86.85185114414142,
      "score_before_time_limit": 86.85185114414142,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3351206434316354,
      "brier": 0.14817864619281185,
      "ece": 0.08688407846981186,
      "n": 373,
      "score": 82.62318430603763,
      "score_before_time_limit": 82.62318430603763,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.956989247311828,
      "interval90_width": 0.5188172043010754,
      "n": 372,
      "nrps": 0.10223206878863621,
      "score": 91.1948723880511,
      "score_before_time_limit": 91.1948723880511,
      "time_limit_failed_share": 0,
      "top_ece": 0.03693524172517095
     }
    },
    "score": 86.88996927941005
   },
   "capability": 63.820192644345624,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 764,418 input + 1,485 output tokens over 1,485 decisions = 515.75959596 tokens/decision. GPU-time cost USD 0.00506801174154553/1,000; old list-price cost USD 0.0155927878787879/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00506801174154553/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.01543686
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005068011741545535
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "JPT-4B (kirp, Qwen3.5-4B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's llm2jev engine on SGLang: one prefill, next-token distribution over the option labels, no generation.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 50.49454797607648,
    "I_sealed": 31.75033894489564,
    "base": 41.12244346048606,
    "excess": 8.904682260825133,
    "gap": 18.744209031180844,
    "penalty": 0.9909531773917487,
    "per_type_split": {
     "open|choice": {
      "cc": 63.186141632614884,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 56.59012268008808,
       "judge": 60.44825313117996,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 40.069013112491376,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 48.2284891831232,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 65.18914834159548,
       "hard": 39.875244434411385,
       "judge": 44.47460178000418,
       "standard": 62.08548020598924
      }
     },
     "sealed|choice": {
      "cc": 57.083332727349514,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 49.52006476234531,
       "judge": 55.92556317335946,
       "standard": 65.82999367222105
      }
     },
     "sealed|noul": {
      "cc": 8.07096660416269,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 34.48275862068966,
       "hard": 5.454545454545445,
       "judge": -10.769230769230765,
       "standard": 28.358208955223873
      }
     },
     "sealed|score": {
      "cc": 30.096717503174723,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 44.10049974856993,
       "hard": 22.500740414784172,
       "judge": 31.671419285857695,
       "standard": 35.92472788323378
      }
     }
    }
   },
   "jevbench_score": 44.19584526267103,
   "key": "jpt-4b",
   "last_measured_on": "2026-10-16",
   "licence": "CC-BY-NC-4.0 (weights, non-commercial)",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "kirp/jpt-4b@05285d2d4f0cd954666ed88095f511c80665bc4c",
   "noul_decisive": {
    "acc_among_decisive": 0.8991935483870968,
    "decisive": 248,
    "decisive_rate": 0.6648793565683646,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.88996927941005,
     "cost": 64.21228687378368,
     "intelligence": 40.7504160092812,
     "speed": 87.95549105905897
    },
    "board_release": "v1.6.43",
    "capability": 63.820192644345624,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 42,
    "capability_rank": 65,
    "composite_ci95": {
     "A": [
      30.849201560578464,
      51.04227850724084
     ],
     "B": [
      27.250163615926105,
      46.552190181580386
     ],
     "C": [
      21.4230566392906,
      35.44602674113947
     ]
    },
    "jevbench_score": 42.17979190383397,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 49,
    "rank": 76,
    "ranks": {
     "A": 76,
     "B": 82,
     "C": 80,
     "capability": 107
    },
    "scores": {
     "A": 42.17979190383397,
     "B": 37.943136667208464,
     "C": 29.29152215544026
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.27590461224317553,
     "p50_s_raw": 0.06295230612158775,
     "p95_s_adjusted": 0.5803525988012552,
     "p95_s_raw": 0.2151762994006276
    },
    "views": {
     "Balanced 33:33:33": 38.706768663218696,
     "Emphasis on Accuracy 60:20:20": 33.02644805072411,
     "Emphasis on Cost 20:20:60": 40.19401225266592,
     "Emphasis on Speed 20:60:20": 44.74726776912738,
     "Intelligence only": 27.06799772975727
    }
   },
   "rank": 135,
   "ranked": true,
   "ranks": {
    "A": 135,
    "B": 142,
    "C": 140,
    "capability": 138
   },
   "repo": "https://huggingface.co/kirp/jpt-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 44.19584526267103,
    "B": 39.230996818536454,
    "C": 30.691559210188217
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2816717634908855,
    "p50_s_raw": 0.06583588174544275,
    "p95_s_adjusted": 0.5873710740823298,
    "p95_s_raw": 0.21868553704116492
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7272727272727273,
       "cc": 45.51030495110692,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100,
        "score": -21.802418480012587
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 45.51030495110692
      },
      "calibration_gen": {
       "accuracy": 0.40594059405940597,
       "cc": 6.942600718140113,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": -12,
        "score": 12.361206184112806
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 6.942600718140113
      },
      "composite_gen": {
       "accuracy": 0.6138613861386139,
       "cc": 30.524274653794333,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -11.111111111111116,
        "score": 35.204260275746144
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 30.524274653794333
      },
      "fairness": {
       "accuracy": 0.8541666666666666,
       "cc": 80.0574932650096,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 80.17247979502879
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 80.0574932650096
      },
      "jaggedness": {
       "accuracy": 0.4722222222222222,
       "cc": 8.716662462559588,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": -18.918918918918916,
        "score": 2.8167407628055585
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 8.716662462559588
      },
      "llm_router": {
       "accuracy": 0.7699680511182109,
       "cc": 64.43530352033419,
       "cc_by_type": {
        "choice": 63.546711279387154,
        "noul": 71.13402061855669,
        "score": 58.62517866305872
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 64.43530352033419
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -19.145184391822436,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -75,
        "score": 16.533518989481156
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6690909090909091,
       "cc": 40.9417458426497,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": 11.111111111111116,
        "score": 45.949535611490745
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 40.9417458426497
      },
      "negation_pair_gen": {
       "accuracy": 0.5813953488372093,
       "cc": 17.585802811544415,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -26.66666666666667,
        "score": 6.090741767966568
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 17.585802811544415
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.796875,
       "cc": 75.72834158389229,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100,
        "score": 57.2150547817069
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 75.72834158389229
      },
      "usecase": {
       "accuracy": 0.5545454545454546,
       "cc": 30.51414077961618,
       "cc_by_type": {
        "choice": 58.33333333333336,
        "noul": 0,
        "score": 33.20908900551518
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 30.51414077961618
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 13.779892768838891,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50,
        "score": -13.422226455388087
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 13.779892768838891
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 57.81691011189755,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 40.117397002359326
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 57.81691011189755
      },
      "de": {
       "accuracy": 0.875,
       "cc": 74.83045721744055,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 75,
        "score": 65.4205751921447
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 74.83045721744055
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 32.71944928714865,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -33.333333333333336,
        "score": 98.53078734003067
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 32.71944928714865
      },
      "en": {
       "accuracy": 0.6159860990443093,
       "cc": 36.19377416348588,
       "cc_by_type": {
        "choice": 57.823311895144045,
        "noul": 19.424460431654666,
        "score": 31.33355016365892
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 36.19377416348588
      },
      "es": {
       "accuracy": 0.9354838709677419,
       "cc": 75.00594678954843,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 34.805768590505025
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 75.00594678954843
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 59.045268832756356,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 76.58935841083739
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 59.045268832756356
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 48.18668967185474,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 65.7365396037995
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 48.18668967185474
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 11.105436929386585,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -50,
        "score": 33.5956403970983
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 11.105436929386585
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 69.7459706377814,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 9.237911913344199
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 69.7459706377814
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 29.893084358860033,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 19.999999999999996,
        "score": 41.16455887483268
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 29.893084358860033
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -2.930607641753516,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": -8.79182292526055
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 93.59160328181015,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 97.44147651209715
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 93.59160328181015
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 19.559769597334363,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 43.960694073388375
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 19.559769597334363
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 27.11064787261347,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -19.999999999999996,
        "score": 24.18425000802239
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 27.11064787261347
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 62.70420351353226,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 59.93347088237638
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 62.70420351353226
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 21.200430571063105,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 60.19703639404037
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 21.200430571063105
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 35.867549234439366,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50,
        "score": 1.08090857288331
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 35.867549234439366
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 70.75565094393932,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 76.78308186407601
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 70.75565094393932
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -2.0732738607933356,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -22.039030621928024
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6826666666666666,
       "cc": 59.011844849690966,
       "cc_by_type": {
        "choice": 59.011844849690966
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 59.011844849690966
      },
      "noul": {
       "accuracy": 0.5946666666666667,
       "cc": 18.933333333333337,
       "cc_by_type": {
        "noul": 18.933333333333337
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 18.933333333333337
      },
      "score": {
       "accuracy": 0.552,
       "cc": 34.07730164309798,
       "cc_by_type": {
        "score": 34.07730164309798
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 34.07730164309798
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -8.588501027763224,
       "cc_by_type": {
        "choice": 36,
        "noul": -75,
        "score": 13.234496916710325
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 25.604935975995005,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 33.33333333333333,
        "score": -0.6179042873358798
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 25.604935975995005
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 62.01973277404002,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 18.5795235253721
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 62.01973277404002
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -21.240602652279154,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": 6.453630639653774
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 20.612938409878996,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 38.65040943253555
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 20.612938409878996
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -43.63239240289681,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -66.66666666666667,
        "score": -113.91604513321872
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 51.14704373786173,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 39.99575306232466
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 51.14704373786173
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -8.074338669114693,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100,
        "score": 38.49475402749912
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 11.222550992581288,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 56.16765297774386
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 11.222550992581288
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": 10.878008756922313,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 0,
        "score": 9.72182070331511
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 10.878008756922313
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 39.76323274592266,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 11.111111111111116,
        "score": 19.80649410340106
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 39.76323274592266
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -37.43831056220761,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": 16.717326377893283
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 24.3873817264206,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -19.999999999999996,
        "score": 27.711089517073706
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 24.3873817264206
      },
      "model_routing": {
       "accuracy": 0.815668202764977,
       "cc": 67.39221756780174,
       "cc_by_type": {
        "choice": 77.7922077922078,
        "noul": 78.78787878787878,
        "score": 45.59656612331862
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 67.39221756780174
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 0.5110011263103331,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 47.59826249601353
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.5110011263103331
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 49.16687512897187,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 47.59347589759341
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 49.16687512897187
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 22.506551648724535,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50,
        "score": 47.076305192479005
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 22.506551648724535
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 20.443709196711993,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 0,
        "score": 19.302142082889596
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 20.443709196711993
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 92.90014978449283,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 89.37660593710837
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 92.90014978449283
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.5481127516258,
      87.7665054437864
     ],
     "I": [
      35.92555449500924,
      44.16081939325531
     ],
     "capability": [
      58.87655522688167,
      65.22805681139722
     ],
     "composite": null,
     "seed": 38
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.88996927941005,
     "capability": 63.820192644345624,
     "intelligence": 40.7504160092812
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6826666666666666,
      "calibration": 86.85185114414142,
      "cc": 59.011844849690966,
      "cc_by_type": {
       "choice": 59.011844849690966
      },
      "cc_tiered": 60.134737179982196,
      "cc_tiered_open": 63.186141632614884,
      "cc_tiered_sealed": 57.083332727349514,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.011844849690966
     },
     "noul": {
      "accuracy": 0.5946666666666667,
      "calibration": 82.62318430603763,
      "cc": 18.933333333333337,
      "cc_by_type": {
       "noul": 18.933333333333337
      },
      "cc_tiered": 24.069989858327034,
      "cc_tiered_open": 40.069013112491376,
      "cc_tiered_sealed": 8.07096660416269,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.933333333333337
     },
     "score": {
      "accuracy": 0.552,
      "calibration": 91.1948723880511,
      "cc": 34.07730164309798,
      "cc_by_type": {
       "score": 34.07730164309798
      },
      "cc_tiered": 39.16260334314896,
      "cc_tiered_open": 48.2284891831232,
      "cc_tiered_sealed": 30.096717503174723,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 34.07730164309798
     }
    },
    "raw_on_A": {
     "calibration": 86.19822310825856,
     "capability": 63.499626434412406,
     "intelligence": 40.80102976056625
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 40.99484800044236,
    "Emphasis on Accuracy 60:20:20": 33.99789716484482,
    "Emphasis on Cost 20:20:60": 44.897985038677284,
    "Emphasis on Speed 20:60:20": 46.52541444235714,
    "Intelligence only": 27.06799772975727
   }
  },
  {
   "adapter_id": "images/jqv/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "hjmurmur (Octalab)",
   "axes": {
    "calibration": 89.87138320201578,
    "cost": 54.72399117000985,
    "intelligence": 33.85815624201783,
    "speed": 82.47453178010326
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06281808353596482,
      "ece_score": 87.43638329280702,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 87.43638329280702,
      "score_before_time_limit": 87.43638329280702,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.34231805929919135,
      "brier": 0.11806081980479606,
      "ece": 0.04224066622806287,
      "n": 371,
      "score": 91.55186675438742,
      "score_before_time_limit": 91.55186675438742,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.959349593495935,
      "interval90_width": 0.5268744354110209,
      "n": 369,
      "nrps": 0.10409415210041804,
      "score": 90.6258995588529,
      "score_before_time_limit": 90.6258995588529,
      "time_limit_failed_share": 0,
      "top_ece": 0.04169392836126208
     }
    },
    "score": 89.87138320201578
   },
   "capability": 61.8647697220168,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 32B active parameters; throughput 9925.03125 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 861,145 input + 0 output tokens over 1,624 decisions = 530.261699507 tokens/decision. GPU-time cost USD 0.0322998342964118/1,000; old list-price cost USD 0.0424209359605911/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0322998342964118/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.032299834296411846
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jqv (Qwen3-32B zero-shot)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 33.07530766335527,
    "I_sealed": 34.641004820680386,
    "base": 33.85815624201783,
    "excess": -4.139084799763362,
    "gap": -1.5656971573251184,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 58.31263706393481,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 47.153192827933324,
       "judge": 64.40342781806196,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 11.77087646652864,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 4.347826086956519,
       "judge": -19.999999999999996,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 29.142409459602362,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 31.012596729013953,
       "hard": 34.8410871612874,
       "judge": 43.10234913255198,
       "standard": -4.129949087897944
      }
     },
     "sealed|choice": {
      "cc": 53.61923821279086,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 62.75219865494053,
       "hard": 49.60407239819003,
       "judge": 55.30726256983241,
       "standard": 54.55105308535534
      }
     },
     "sealed|noul": {
      "cc": 17.979675703685576,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 60.00000000000001,
       "hard": -4.854368932038833,
       "judge": 21.052631578947366,
       "standard": 38.0281690140845
      }
     },
     "sealed|score": {
      "cc": 32.32410054556472,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 52.65739446915423,
       "hard": 22.31408478189991,
       "judge": 37.34025956092481,
       "standard": 34.65324658805946
      }
     }
    }
   },
   "jevbench_score": 25.8118135489931,
   "key": "jqv",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (Qwen3-32B weights); serving code public",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9385245901639344,
    "decisive_rate": 0.6576819407008087,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.87138320201578,
     "cost": 51.172592607797476,
     "intelligence": 33.85815624201783,
     "speed": 83.1032423977654
    },
    "board_release": "v1.6.43",
    "capability": 61.8647697220168,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 52,
    "capability_rank": 77,
    "composite_ci95": {
     "A": [
      15.505551311407853,
      35.88495273799785
     ],
     "B": [
      13.30350456735459,
      32.67206876334683
     ],
     "C": [
      10.767743966255452,
      24.920106068054068
     ]
    },
    "jevbench_score": 25.391439220985372,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 77,
    "rank": 110,
    "ranks": {
     "A": 110,
     "B": 113,
     "C": 125,
     "capability": 120
    },
    "scores": {
     "A": 25.391439220985372,
     "B": 22.528313466257792,
     "C": 17.63294390346206
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3448032220825553,
     "p50_s_raw": 0.09740161104127765,
     "p95_s_adjusted": 1.4193988358601928,
     "p95_s_raw": 0.6346994179300964
    },
    "views": {
     "Balanced 33:33:33": 22.511071395625475,
     "Emphasis on Accuracy 60:20:20": 19.077648593308684,
     "Emphasis on Cost 20:20:60": 22.883252890033052,
     "Emphasis on Speed 20:60:20": 26.917672025192456,
     "Intelligence only": 15.525654079176814
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 175,
   "ranked": true,
   "ranks": {
    "A": 175,
    "B": 176,
    "C": 177,
    "capability": 150
   },
   "repo": "https://github.com/Octalab-Inc/jqv",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 25.8118135489931,
    "B": 22.791781106358858,
    "C": 17.924870520134096
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3853486230131239,
    "p50_s_raw": 0.11767431150656193,
    "p95_s_adjusted": 1.4678883056156335,
    "p95_s_raw": 0.6589441528078168
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3-32B, bf16, read as a direct-logit classifier (no fine-tuning)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.42862757082423,
      89.91697762368707
     ],
     "I": [
      27.681410439375405,
      39.196898115223256
     ],
     "capability": [
      56.87716097755697,
      63.63771518645792
     ],
     "composite": null,
     "seed": 51
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.87138320201578,
     "capability": 61.8647697220168,
     "intelligence": 33.85815624201783
    },
    "per_type": {
     "choice": {
      "accuracy": 0.648,
      "calibration": 87.43638329280702,
      "cc": 54.98422742817084,
      "cc_by_type": {
       "choice": 54.98422742817084
      },
      "cc_tiered": 55.96593763836283,
      "cc_tiered_open": 58.31263706393481,
      "cc_tiered_sealed": 53.61923821279086,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.98422742817084
     },
     "noul": {
      "accuracy": 0.6106666666666667,
      "calibration": 91.55186675438742,
      "cc": 22.13333333333334,
      "cc_by_type": {
       "noul": 22.13333333333334
      },
      "cc_tiered": 14.875276085107107,
      "cc_tiered_open": 11.77087646652864,
      "cc_tiered_sealed": 17.979675703685576,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 22.13333333333334
     },
     "score": {
      "accuracy": 0.5066666666666667,
      "calibration": 90.6258995588529,
      "cc": 32.28871510053675,
      "cc_by_type": {
       "score": 32.28871510053675
      },
      "cc_tiered": 30.73325500258354,
      "cc_tiered_open": 29.142409459602362,
      "cc_tiered_sealed": 32.32410054556472,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 32.28871510053675
     }
    },
    "raw_on_A": {
     "calibration": 85.0933345598925,
     "capability": 62.946394387426395,
     "intelligence": 40.79945421496029
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 22.952950817217538,
    "Emphasis on Accuracy 60:20:20": 19.26624887924249,
    "Emphasis on Cost 20:20:60": 23.763864318793093,
    "Emphasis on Speed 20:60:20": 27.235183843288397,
    "Intelligence only": 15.525654079176814
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Julien Lestel / @jlestel (JuL)",
   "axes": {
    "calibration": 77.79406843975872,
    "cost": 68.9870979950853,
    "intelligence": 7.4645122320968404,
    "speed": 90.5592978634667
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.049032187499999956,
      "ece_score": 90.19356250000001,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 90.19356250000001,
      "score_before_time_limit": 90.19356250000001,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.7013333333333334,
      "brier": 0.20025538554666658,
      "ece": 0.08354026666666664,
      "n": 375,
      "score": 83.29194666666668,
      "score_before_time_limit": 83.29194666666668,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8418230563002681,
      "interval90_width": 0.7091152815013402,
      "n": 373,
      "nrps": 0.22054212385388733,
      "score": 59.89669615260948,
      "score_before_time_limit": 60.70611096548258,
      "time_limit_failed_share": 0.013333333333333334,
      "top_ece": 0.2826678284182305
     }
    },
    "score": 77.79406843975872
   },
   "capability": 42.62929033592778,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.23984046541310475,
     1.1184890962978362
    ],
    "B": [
     0.16476538317869,
     0.8100368206640075
    ],
    "C": [
     0.16655587875910055,
     0.7767285390957197
    ]
   },
   "cost": {
    "basis": "ESTIMATE (size-class reference; self-hosted open weights, no bookable listing for this base). openbmb/MiniCPM5-2B is an EXPLICIT entry in the frozen module's UNLISTED_BASE_MODELS table, so pricing_v15.reference() returns None without raising: the decision that there is no bookable listing for this base is already taken, and the only MiniCPM key in either dated snapshot is deepinfra:openbmb/MiniCPM-Llama3-V-2_5 at (0.34, 0.34), a VISION model and not a comparable reference. The row therefore carries its own labelled size-class estimate: deepinfra:Qwen/Qwen3.5-2B at USD 0.02 per 1M input and USD 0.10 per 1M output in the frozen 25 Sep snapshot - a 2B dense instruct model, the same size class as this 2B base - x this row's own measured tokens. Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, not by estimate: the readout is a distribution over candidate labels from one forward pass and nothing is generated, so the row's cost rests on the input rate alone and any other decision is reversible by arithmetic from the raw. PRICING REFERENCE NOTE: if a bookable MiniCPM5-2B text listing has appeared since 25 Sep, this row reprices by one multiplication from the raw. receipts/COST-ALTERNATIVES-R58.json has the arithmetic for the alternatives already done.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.016866253333333334
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.010808449559918753
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 5,
    "n_over_limit_answered_ok": 4,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "jul fast (usejul/jul-decision-minicpm5-2b: MiniCPM5-2B fine-tune with a pointer head, option probabilities from one pass, self-hosted via `jul serve`)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,498 of 1,500 answered. Library github.com/usejul/jul tag v0.5.1, PyPI jul==0.5.1, Apache-2.0; weights usejul/jul-decision-minicpm5-2b at f556a6936089693ea68bfed3080c637e4853a3ff, Apache-2.0; base openbmb/MiniCPM5-2B at 12a3808a. NO MAPPING DECISION OF OURS: `jul serve` speaks the TypeSafe /v1/systemone wire format, so the UNCHANGED jevbench.adapters.typesafe applies, and all eight checks of the unamended smoke gate passed on synthetic requests before any sealed item was sent. STACK: the author's own published install line from their request, `pip install \"jul[torch]==0.5.1\"` in an isolated venv (torch 2.14.1+cu130, transformers 5.19.0), served with their own published command verbatim (`jul serve --model jul-decision-minicpm5-2b --host 127.0.0.1 --port 8577`). One RTX 5090 32 GB (Lium, Paris FR). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW IS THE FIRST THAT NEITHER REFUSES NOR UNIFORMLY TRUNCATES THEM, AND THE SPLIT IS BY QUESTION TYPE. The author declares a 2,048-token state budget with no refusal - 'the beginning of the text is kept and the end is cut, without an error'. Their server's own usage.input_tokens shows that budget applied to 17 of the 23 (counts 2,096-2,135) and NOT to the other 6, which were read at 107,331-107,997 tokens; those 6 are exactly the 6 score questions among the 23, while all 17 truncated ones are choice or noul. Two of the six full reads then failed with the server's own CUDA OutOfMemoryError on a 32 GB card ('Tried to allocate 5.59 GiB' / '5.41 GiB'), which is this row's only 2 invalid and which a larger card would most likely have answered: the cause is the score path not applying the author's own documented truncation, and the author's note that '24 GB is enough' holds only under that truncation. The 2 invalid are counted wrong, the conservative direction, and are 0.13 % of the pool. PROBABILITIES ARE RETURNED ROUNDED TO FOUR DECIMALS: 426 of 1,498 answered distributions sum to 1 +/- 2e-4 (observed range 0.9998-1.0002); none is off by more than that, so it is the server's rounding and not a partial readout. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 58, GitHub fstandhartinger/jevbench#198",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 7.349597785148552,
    "I_sealed": 7.579426679045129,
    "base": 7.4645122320968404,
    "excess": -2.8032165363348214,
    "gap": -0.22982889389657757,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 22.048793355445657,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.945945945945947,
       "hard": 15.06763133060712,
       "judge": 16.941331575477918,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -100,
       "judge": -50,
       "standard": -55.55555555555556
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 26.921515151515163,
       "hard": -16.06981481481484,
       "judge": 27.538983050847456,
       "standard": -57.756296296296284
      }
     },
     "sealed|choice": {
      "cc": 22.73828003713539,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 54.77052693814206,
       "hard": 14.479638009049756,
       "judge": 25.51210428305402,
       "standard": 19.078704273925375
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -4.0000000000000036,
       "hard": -74.75728155339806,
       "judge": -42.10526315789473,
       "standard": -32.3943661971831
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 2.8917730496454075,
       "hard": -23.2233823529411,
       "judge": -11.343473053892229,
       "standard": -27.880304568527947
      }
     }
    }
   },
   "jevbench_score": 0.5172342787655929,
   "key": "jul-fast",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "usejul/jul-decision-minicpm5-2b@f556a6936089693ea68bfed3080c637e4853a3ff",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8928571428571429,
    "decisive_rate": 0.2986666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.06387337738309,
     "cost": 68.9870979950853,
     "intelligence": 7.4645122320968404,
     "speed": 90.69403874154062
    },
    "board_release": "v1.6.43",
    "capability": 42.76419280473996,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 108,
    "capability_rank": 136,
    "composite_ci95": {
     "A": [
      0.2399155159563835,
      1.1190632145905357
     ],
     "B": [
      0.16479417918823877,
      0.8102776687445344
     ],
     "C": [
      0.166607997191933,
      0.7771272323545388
     ]
    },
    "jevbench_score": 0.5174168972906218,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 161,
    "rank": 204,
    "ranks": {
     "A": 204,
     "B": 205,
     "C": 204,
     "capability": 199
    },
    "scores": {
     "A": 0.5174168972906218,
     "B": 0.3638597737908031,
     "C": 0.35931728978515404
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23712351802387274,
     "p50_s_raw": 0.043561759011936374,
     "p95_s_adjusted": 0.3594359368114965,
     "p95_s_raw": 0.10471796840574825
    },
    "views": {
     "Balanced 33:33:33": 0.41923158593664034,
     "Emphasis on Accuracy 60:20:20": 0.26072023907855174,
     "Emphasis on Cost 20:20:60": 0.5912460853155722,
     "Emphasis on Speed 20:60:20": 0.613844381562066,
     "Intelligence only": 0.16636589222451567
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 246,
   "ranked": true,
   "ranks": {
    "A": 246,
    "B": 247,
    "C": 246,
    "capability": 240
   },
   "repo": "https://huggingface.co/usejul/jul-decision-minicpm5-2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.5172342787655929,
    "B": 0.363787515323243,
    "C": 0.35919047136499505
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2375775480235461,
    "p50_s_raw": 0.04378877401177306,
    "p95_s_adjusted": 0.3700537570839515,
    "p95_s_raw": 0.11002687854197575
   },
   "status": {
    "answered_ok": 1498,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "openbmb/MiniCPM5-2B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.94462322454935,
      80.16649632842498
     ],
     "I": [
      5.676467601887915,
      9.888156546142561
     ],
     "capability": [
      39.70355516328919,
      44.189161649231124
     ],
     "composite": {
      "A": [
       0.23984046541310475,
       1.1184890962978362
      ],
      "B": [
       0.16476538317869,
       0.8100368206640075
      ],
      "C": [
       0.16655587875910055,
       0.7767285390957197
      ]
     },
     "seed": 74
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.79406843975872,
     "capability": 42.62929033592778,
     "intelligence": 7.4645122320968404
    },
    "per_type": {
     "choice": {
      "accuracy": 0.404,
      "calibration": 90.19356250000001,
      "cc": 23.78011234997108,
      "cc_by_type": {
       "choice": 23.78011234997108
      },
      "cc_tiered": 22.393536696290525,
      "cc_tiered_open": 22.048793355445657,
      "cc_tiered_sealed": 22.73828003713539,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 23.78011234997108
     },
     "noul": {
      "accuracy": 0.26666666666666666,
      "calibration": 83.29194666666668,
      "cc": -46.666666666666664,
      "cc_by_type": {
       "noul": -46.666666666666664
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2826666666666667,
      "calibration": 59.89669615260948,
      "cc": -15.119236842105233,
      "cc_by_type": {
       "score": -15.119236842105233
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 78.08896065787587,
     "capability": 43.460037896212135,
     "intelligence": 8.831115134548394
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.4191884671241191,
    "Emphasis on Accuracy 60:20:20": 0.2607102324554309,
    "Emphasis on Cost 20:20:60": 0.5911946272474554,
    "Emphasis on Speed 20:60:20": 0.613678011636167,
    "Intelligence only": 0.16636589222451567
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Supersonic Labs / @supersonicai (Hugging Face SupersonicLabs)",
   "axes": {
    "calibration": 17.26803648238681,
    "cost": 89.74859303626371,
    "intelligence": 2.4412732607743255,
    "speed": 77.74663940674506
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.49162634593303195,
      "ece_score": 1.67473081339361,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 304,
      "n_tvd": 0,
      "score": 1.67473081339361,
      "score_before_time_limit": 1.67473081339361,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.27419354838709675,
      "brier": 0.45131804975332956,
      "ece": 0.4322726807691505,
      "n": 372,
      "score": 13.545463846169902,
      "score_before_time_limit": 13.545463846169902,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.5887096774193549,
      "interval90_width": 0.3958333333333335,
      "n": 372,
      "nrps": 0.26832170424806157,
      "score": 36.583914787596925,
      "score_before_time_limit": 36.583914787596925,
      "time_limit_failed_share": 0,
      "top_ece": 0.5187092957892186
     }
    },
    "score": 17.26803648238681
   },
   "capability": 9.854654871580568,
   "composite_ci95": {
    "A": [
     0.0007030229261073844,
     0.1394909016822168
    ],
    "B": [
     0.0004538884390108904,
     0.10093093032471417
    ],
    "C": [
     0.0004882103653523503,
     0.09686868172376169
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no public bookable tariff for these weights. The frozen pricing_v15 path is asserted with ONE authorised, scoped correction: the frozen pricing update added the declared base jhu-clsp/mmBERT-small to pricing_v15.UNLISTED_BASE_MODELS, exactly as openbmb/MiniCPM5-2B and answerdotai/ModernBERT-base already are, so reference() returns no market reference for it rather than raising and the ancestry can be carried in base_model instead of being dropped to null. The addition changes no other row: it was proved behaviour-identical over every registry entry on the box before it was installed (r108/receipts/PRICING-EQUIV-R108.json) and independently reviewed (r108/receipts/REVIEW-PRICING-R108.json). The published 24 Sep estimate rule then gives a clearly labelled size-class estimate: the base-size-encoder line the PUBLISHED v1.6 field already carries - USD 0.005 per 1M input and 0.0 per 1M output, the line modernbert-jev-v2, tacet-sonata and watt-flash-0.1 carry there (no row of THIS draw carries it, and lev-350m carries 0.01/0.0, not this line) (four base-size encoders in the dated 25 Sep 2026 DeepInfra snapshot sha256 84eb7926... all price at exactly that) - so the frozen pricing reference is unchanged. x this row's own measured tokens: usage.input_tokens comes from the author's OWN audit function engine.encoding_info() and usage.output_tokens is 0 by construction (one encoder forward pass read at marker positions, nothing generated). Estimated, not charged. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.002196 per 1,000 decisions (0.034x the official cost cap USD 0.064595), against USD 0.002174 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0021744466666666668
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0021964107744107744
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Julia-1 (Supersonic Labs)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Sandy, the evaluator's own shared CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2), NOT a dedicated machine: other jobs ran beside it and the 1-minute load average was logged every 60 s into logs/CONDITIONS-julia-1.jsonl (11 samples, 10.87-13.78, mean 12.5). The author ships NO server, so the HTTP surface is OURS: r107/src/julia_shim_r107.py, a pure transport that makes no mapping decision, because the author's own julia.typed.predict_typed already speaks JevBench's typed-question shape exactly (choice keyed by the item's own criteria keys, score keyed '0'..'n-1' with score = sum(i*p_i), noul = p(true)). Engine built with the author's own inference-policy.json (max_length 8192, head_length 512, strict_encoding true), torch backend, device cpu, float32 (their julia_config weight_dtype), thread count left at their own default of 4 - all four read back from the run's own /health rather than asserted. Run inside a loopback-only network namespace whose outbound probe failed as required, offline (HF_HUB_OFFLINE=1), credential-free, nice 5, in a systemd scope with MemoryMax=8G and no swap. Speed is therefore a loaded-shared-CPU figure and a lower bound for this model on a dedicated core or a GPU; the standard self-hosted x2 + 0.15 s adjustment is applied on top and the raw p50/p95 are kept. Measured for this release on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...) under the pre-registration, which admitted this row only after an independent source review and a synthetic-only smoke, both of which were met in that order before the first sealed item. DISCLOSED: the author's own strict_encoding policy REFUSES rather than truncates, so 17 of the 1,500 items (15 over their lossless context budget, 2 with an option above their 48-token option contract) are honest item-level misses recorded as HTTP 422 - predicted exactly, class split included, before the run.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 2.994076934245856,
    "I_sealed": 1.888469587302795,
    "base": 2.4412732607743255,
    "excess": -8.733919423412651,
    "gap": 1.105607346943061,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 8.982230802737568,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 2.7027027027027115,
       "hard": 13.180245360176164,
       "judge": 5.075807514831907,
       "standard": 9.585600669736293
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -4.347826086956519,
       "judge": 0,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -4.595553956916043,
       "hard": -21.831788639496995,
       "judge": -76.77226748673378,
       "standard": -19.64084658486791
      }
     },
     "sealed|choice": {
      "cc": 5.6654087619083855,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 12.325285895806868,
       "hard": 0.6013646351335588,
       "judge": 11.85112634671892,
       "standard": 3.1849820712929935
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -41.379310344827594,
       "hard": -41.81818181818182,
       "judge": -50.76923076923077,
       "standard": -19.402985074626866
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": -36.27109595021023,
       "hard": -40.51450244498809,
       "judge": -18.32435058181421,
       "standard": -13.222659125532354
      }
     }
    }
   },
   "jevbench_score": 0.01939975687732831,
   "key": "julia-1",
   "last_measured_on": "2026-10-13",
   "licence": "Apache-2.0 (Hugging Face cardData.license and README; the pinned tree contains no LICENSE file)",
   "listing": "ranked",
   "measured_in": "v1.6.15",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "SupersonicLabs/Julia-1@a85b127321d580d65176c89ced8273f305745d85",
   "noul_decisive": {
    "acc_among_decisive": 0.45555555555555555,
    "decisive": 270,
    "decisive_rate": 0.7258064516129032,
    "n": 375,
    "supported": true,
    "valid": 372
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 17.26803648238681,
     "cost": 89.74859303626371,
     "intelligence": 2.4412732607743255,
     "speed": 79.42119248319958
    },
    "board_release": "v1.6.43",
    "capability": 9.854654871580568,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 138,
    "capability_rank": 168,
    "composite_ci95": {
     "A": [
      0.0007031616572555455,
      0.13962672310195573
     ],
     "B": [
      0.000453934696458772,
      0.10098779426488583
     ],
     "C": [
      0.0004883067064274621,
      0.09696300215413593
     ]
    },
    "jevbench_score": 0.019410466159695165,
    "measured_in": "v1.6.15",
    "method": "v1.6",
    "open_board_rank": 186,
    "rank": 230,
    "ranks": {
     "A": 230,
     "B": 230,
     "C": 230,
     "capability": 240
    },
    "scores": {
     "A": 0.019410466159695165,
     "B": 0.013230972842696788,
     "C": 0.0134794903886772
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.7337018699967303,
     "p50_s_raw": 0.2918509349983651,
     "p95_s_adjusted": 1.5572599903855007,
     "p95_s_raw": 0.7036299951927504
    },
    "views": {
     "Balanced 33:33:33": 0.016503253650422215,
     "Emphasis on Accuracy 60:20:20": 0.009515909044619377,
     "Emphasis on Cost 20:20:60": 0.026160186027009726,
     "Emphasis on Speed 20:60:20": 0.02599486708790751,
     "Intelligence only": 0.005819814930094012
    }
   },
   "rank": 271,
   "ranked": true,
   "ranks": {
    "A": 271,
    "B": 271,
    "C": 271,
    "capability": 281
   },
   "repo": "https://huggingface.co/SupersonicLabs/Julia-1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.01939975687732831,
    "B": 0.013226991120613374,
    "C": 0.01347205338703355
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.8151886820036452,
    "p50_s_raw": 0.3325943410018226,
    "p95_s_adjusted": 2.0609996498023975,
    "p95_s_raw": 0.9554998249011987
   },
   "status": {
    "answered_ok": 1483,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.2727272727272727,
       "cc": -27.879433144127862,
       "cc_by_type": {
        "choice": 0,
        "noul": 0,
        "score": -83.63829943238359
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.3564356435643564,
       "cc": -3.761010359494321,
       "cc_by_type": {
        "choice": 4.559915164369044,
        "noul": 4.0000000000000036,
        "score": -19.842946242852012
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.25742574257425743,
       "cc": -31.644341476391443,
       "cc_by_type": {
        "choice": -7.317073170731707,
        "noul": -18.518518518518523,
        "score": -69.0974327399241
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.3541666666666667,
       "cc": -5.944000384605097,
       "cc_by_type": {
        "choice": 4.761904761904761,
        "noul": -60,
        "score": 37.40609408427995
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.37222222222222223,
       "cc": -8.884037970016676,
       "cc_by_type": {
        "choice": 24.927815206929747,
        "noul": -45.945945945945944,
        "score": -5.6339831710338295
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.23003194888178913,
       "cc": -20.437719287542667,
       "cc_by_type": {
        "choice": -0.6427753808223877,
        "noul": -38.144329896907216,
        "score": -22.526052584898395
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.18518518518518517,
       "cc": -32.03827054080279,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -75,
        "score": -22.145739457459925
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.29454545454545455,
       "cc": -25.78754754021827,
       "cc_by_type": {
        "choice": 17.835017932833402,
        "noul": -55.55555555555556,
        "score": -39.64210499793266
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.3372093023255814,
       "cc": -37.503184879754805,
       "cc_by_type": {
        "choice": 6.666666666666665,
        "noul": -13.33333333333333,
        "score": -105.84288797259775
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.20833333333333334,
       "cc": -1.1984021304926722,
       "cc_by_type": {
        "choice": -1.1984021304926722
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 0
      },
      "perturb_pair": {
       "accuracy": 0.265625,
       "cc": -25.52938809242961,
       "cc_by_type": {
        "choice": -5.105105105105101,
        "noul": 19.999999999999996,
        "score": -91.48305917218373
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.2409090909090909,
       "cc": -16.515408619322642,
       "cc_by_type": {
        "choice": 3.1862745098039587,
        "noul": -25,
        "score": -27.732500367771884
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -2.776462098911864,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0,
        "score": -51.186529153878446
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.14285714285714285,
       "cc": 36.84210526315789,
       "cc_by_type": {
        "choice": -26.315789473684216,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 36.84210526315789
      },
      "da": {
       "accuracy": 0.3076923076923077,
       "cc": -37.718345339022456,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -100,
        "score": -50.9430544502471
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.4375,
       "cc": 22.354227161079496,
       "cc_by_type": {
        "choice": 20.353982300884955,
        "noul": 25,
        "score": 21.70869918235354
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 22.354227161079496
      },
      "el": {
       "accuracy": 0.2,
       "cc": 0.6000833683740154,
       "cc_by_type": {
        "choice": -17.31843575418994,
        "noul": -33.333333333333336,
        "score": 52.452019192645324
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0.6000833683740154
      },
      "en": {
       "accuracy": 0.2797567332754127,
       "cc": -16.97684900279254,
       "cc_by_type": {
        "choice": 3.7844302607973472,
        "noul": -26.618705035971225,
        "score": -28.096272233203745
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.41935483870967744,
       "cc": -11.423096652088782,
       "cc_by_type": {
        "choice": 31.484502446982063,
        "noul": -25,
        "score": -40.75379240324841
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -39.640298601530766,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": -20.56024006688737
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.21428571428571427,
       "cc": -41.1336147354147,
       "cc_by_type": {
        "choice": 15.294117647058824,
        "noul": -100,
        "score": -38.69496185330294
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.14285714285714285,
       "cc": -67.07248359384337,
       "cc_by_type": {
        "choice": -0.5586592178770929,
        "noul": -100,
        "score": -100.65879156365303
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0,
       "cc": -66.66666666666667,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.3333333333333333,
       "cc": -25.47727987845241,
       "cc_by_type": {
        "choice": 15.822291504286824,
        "noul": -33.333333333333336,
        "score": -58.92079780631072
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.24,
       "cc": -18.87257764229911,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": -80,
        "score": -5.132427128644745
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0,
       "cc": -77.83191986347525,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": -100.16242625709242
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.35714285714285715,
       "cc": -10.156143207655505,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -33.333333333333336,
        "score": -13.801762956299847
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.2,
       "cc": -51.11558643410791,
       "cc_by_type": {
        "choice": -29.87012987012987,
        "noul": -33.333333333333336,
        "score": -90.14329609886053
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.23076923076923078,
       "cc": -31.24251604325812,
       "cc_by_type": {
        "choice": 16.208209902666106,
        "noul": -80,
        "score": -29.93575803244046
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.39285714285714285,
       "cc": 2.5150561249822148,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": 19.999999999999996,
        "score": -38.20615749423426
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 2.5150561249822148
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -37.55166369327866,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100,
        "score": -10.52733150536791
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.38461538461538464,
       "cc": -27.04550827576324,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100,
        "score": -15.919133522941898
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.14285714285714285,
       "cc": -41.81990641185333,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100,
        "score": -28.68552568717291
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.3,
       "cc": -68.47760196419559,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100,
        "score": -129.1616194519088
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.272,
       "cc": 5.968349949291043,
       "cc_by_type": {
        "choice": 5.968349949291043
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 5.968349949291043
      },
      "noul": {
       "accuracy": 0.328,
       "cc": -34.4,
       "cc_by_type": {
        "noul": -34.4
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.24533333333333332,
       "cc": -29.344915811166672,
       "cc_by_type": {
        "score": -29.344915811166672
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -38.4024160564452,
       "cc_by_type": {
        "choice": -12,
        "noul": -75,
        "score": -28.207248169335596
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": -2.238256630096153,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -11.111111111111116,
        "score": -39.70303766116492
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.42857142857142855,
       "cc": -3.7905034661254127,
       "cc_by_type": {
        "choice": 18.699186991869915,
        "noul": 50,
        "score": -80.07069739024615
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -50.406556307064626,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -69.34832389195411
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": -8.733832465640946,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": -33.333333333333336,
        "score": -20.40439594764747
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -52.37905037945314,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": 0,
        "score": -169.08683667295062
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.23333333333333334,
       "cc": -29.56253397544977,
       "cc_by_type": {
        "choice": 21.568627450980394,
        "noul": -71.42857142857143,
        "score": -38.827657948758265
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.09090909090909091,
       "cc": -54.303624656078085,
       "cc_by_type": {
        "choice": -4.529616724738673,
        "noul": -100,
        "score": -58.38125724349559
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.06666666666666667,
       "cc": -27.932241245491838,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -100,
        "score": 20.369942930191154
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -4.197964663778406,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": 0,
        "score": -9.810168081270977
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.24,
       "cc": -7.467987900190109,
       "cc_by_type": {
        "choice": 6.976744186046504,
        "noul": -33.333333333333336,
        "score": 3.9526254467165067
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -26.741444679738834,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": -17.858742641367044
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.2962962962962963,
       "cc": -17.699209031887154,
       "cc_by_type": {
        "choice": 19.385796545105567,
        "noul": -60,
        "score": -12.483423640767022
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.22580645161290322,
       "cc": -17.59559356925139,
       "cc_by_type": {
        "choice": -5.194805194805169,
        "noul": -33.333333333333336,
        "score": -14.258642179615656
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0
      },
      "moderation_trust_safety": {
       "accuracy": 0.16666666666666666,
       "cc": -51.88151472980022,
       "cc_by_type": {
        "choice": -3.646833013435703,
        "noul": -100,
        "score": -51.99771117596499
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.4444444444444444,
       "cc": 0.4935502236767717,
       "cc_by_type": {
        "choice": -0.2785515320334252,
        "noul": 33.33333333333333,
        "score": -31.57413113026959
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0.4935502236767717
      },
      "risk_assessment": {
       "accuracy": 0.1111111111111111,
       "cc": -49.23270679758713,
       "cc_by_type": {
        "choice": -18.226600985221676,
        "noul": -100,
        "score": -29.471519407539716
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.08695652173913043,
       "cc": -38.04052057494357,
       "cc_by_type": {
        "choice": -15.942028985507239,
        "noul": -66.66666666666667,
        "score": -31.5128660726568
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.20833333333333334,
       "cc": -9.065556816294313,
       "cc_by_type": {
        "choice": 3.914590747330961,
        "noul": 0,
        "score": -31.111261196213903
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      13.378936392289322,
      21.888965168274606
     ],
     "I": [
      0.7771519016149122,
      4.933627097063504
     ],
     "capability": [
      7.593761103357901,
      12.813165041600257
     ],
     "composite": {
      "A": [
       0.0007030229261073844,
       0.1394909016822168
      ],
      "B": [
       0.0004538884390108904,
       0.10093093032471417
      ],
      "C": [
       0.0004882103653523503,
       0.09686868172376169
      ]
     },
     "seed": 39
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 17.26803648238681,
     "capability": 9.854654871580568,
     "intelligence": 2.4412732607743255
    },
    "per_type": {
     "choice": {
      "accuracy": 0.272,
      "calibration": 1.67473081339361,
      "cc": 5.968349949291043,
      "cc_by_type": {
       "choice": 5.968349949291043
      },
      "cc_tiered": 7.3238197823229765,
      "cc_tiered_open": 8.982230802737568,
      "cc_tiered_sealed": 5.6654087619083855,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 5.968349949291043
     },
     "noul": {
      "accuracy": 0.328,
      "calibration": 13.545463846169902,
      "cc": -34.4,
      "cc_by_type": {
       "noul": -34.4
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.24533333333333332,
      "calibration": 36.583914787596925,
      "cc": -29.344915811166672,
      "cc_by_type": {
       "score": -29.344915811166672
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 19.70003977436594,
     "capability": 11.004198815490696,
     "intelligence": 2.3083578566154506
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.016492932352696162,
    "Emphasis on Accuracy 60:20:20": 0.00951384924652731,
    "Emphasis on Cost 20:20:60": 0.026144624889269167,
    "Emphasis on Speed 20:60:20": 0.02594882609560486,
    "Intelligence only": 0.005819814930094012
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Axel Montzamir (Okura66)",
   "axes": {
    "calibration": 83.76747077065221,
    "cost": 64.66738128007644,
    "intelligence": 39.71971801341685,
    "speed": 89.23426082115222
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06004736643320625,
      "ece_score": 87.99052671335875,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 87.99052671335875,
      "score_before_time_limit": 87.99052671335875,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2991913746630728,
      "brier": 0.1447023593915565,
      "ece": 0.10598766007071211,
      "n": 371,
      "score": 78.80246798585758,
      "score_before_time_limit": 78.80246798585758,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9214092140921409,
      "interval90_width": 0.3678861788617887,
      "n": 369,
      "nrps": 0.09917884531883965,
      "score": 84.50941761274032,
      "score_before_time_limit": 84.50941761274032,
      "time_limit_failed_share": 0,
      "top_ece": 0.10531640121317691
     }
    },
    "score": 83.76747077065221
   },
   "capability": 61.74359439203453,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,488,569 input + 0 output tokens over 1,624 decisions = 1532.37007389 tokens/decision. GPU-time cost USD 0.0150575376351973/1,000; old list-price cost USD 0.0459711022167488/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0150575376351973/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.015057537635197281
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Kahn1 4B (Okura66, Qwen3.5-4B LoRA merge, author's sysone engine)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-07",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX 5090, the author's own sysone engine (github.com/Okura66/kahn1) on vLLM 0.30.0 with the author's documented max-model-len 4096 and calibration.json, offline (all credentials unset), driven over loopback only by the JevBench client through a thin pre-registered adapter for the author's /v1/evaluate/jev endpoint; label-free inputs; pod destroyed afterwards; JevBench add-request, measured 7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 41.96168150915031,
    "I_sealed": 37.4777545176834,
    "base": 39.71971801341685,
    "excess": 1.910539349028665,
    "gap": 4.483926991466909,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 62.694715037494376,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 69.80182447310474,
       "judge": 52.537903757415954,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 17.904071773636996,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 30.434782608695656,
       "judge": -9.999999999999998,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 45.28625771631957,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.624702928143726,
       "hard": 52.08153129449285,
       "judge": 35.187676087499,
       "standard": 44.67436039729178
      }
     },
     "sealed|choice": {
      "cc": 60.13243945421864,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 69.40359175227256,
       "hard": 56.730769230769226,
       "judge": 57.16945996275605,
       "standard": 66.7446729892844
      }
     },
     "sealed|noul": {
      "cc": 16.581379950628655,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -10.679611650485432,
       "judge": 28.947368421052634,
       "standard": 40.8450704225352
      }
     },
     "sealed|score": {
      "cc": 35.719444148202925,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 55.76848705914599,
       "hard": 35.73652095498605,
       "judge": 30.714342076628732,
       "standard": 33.16842218652643
      }
     }
    }
   },
   "jevbench_score": 39.57470163141661,
   "key": "kahn1",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Okura66/Kahn1-Qwen3.5-4B@f3f9910f995b0e345a6e8a9cb0aeed1e06bafaf6",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8653846153846154,
    "decisive_rate": 0.7008086253369272,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.76747077065221,
     "cost": 50.12545250055582,
     "intelligence": 39.71971801341685,
     "speed": 89.97531755002697
    },
    "board_release": "v1.6.43",
    "capability": 61.74359439203453,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 54,
    "capability_rank": 79,
    "composite_ci95": {
     "A": [
      24.53974660316557,
      46.8147086789254
     ],
     "B": [
      21.806431754904573,
      43.49585293740054
     ],
     "C": [
      17.041490696642754,
      32.51021436036486
     ]
    },
    "jevbench_score": 37.0242246815327,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 59,
    "rank": 89,
    "ranks": {
     "A": 89,
     "B": 87,
     "C": 90,
     "capability": 123
    },
    "scores": {
     "A": 37.0242246815327,
     "B": 33.799163379528274,
     "C": 25.711267139953264
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2821926829987206,
     "p50_s_raw": 0.06609634149936028,
     "p95_s_adjusted": 0.3563875272346194,
     "p95_s_raw": 0.10319376361730971
    },
    "views": {
     "Balanced 33:33:33": 33.66233685744503,
     "Emphasis on Accuracy 60:20:20": 29.601398852667728,
     "Emphasis on Cost 20:20:60": 32.81983249388412,
     "Emphasis on Speed 20:60:20": 40.21102161322423,
     "Intelligence only": 25.065620562020474
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a8 (scorer output sha256 7a5b4d4c7b7c, method option B / O1S)"
   },
   "rank": 149,
   "ranked": true,
   "ranks": {
    "A": 149,
    "B": 149,
    "C": 151,
    "capability": 153
   },
   "repo": "https://huggingface.co/Okura66/Kahn1-Qwen3.5-4B",
   "repriced_in": "v1.6.37",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 39.57470163141661,
    "B": 35.46855090428108,
    "C": 27.482431688483757
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.30520028799073773,
    "p50_s_raw": 0.07760014399536885,
    "p95_s_adjusted": 0.3908309774473309,
    "p95_s_raw": 0.12041548872366546
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "supersedes_base_row": "v1.6.1",
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.07087093913225,
      85.02256483809754
     ],
     "I": [
      33.5350460501225,
      43.90853133816846
     ],
     "capability": [
      56.37025407029597,
      63.949534098181175
     ],
     "composite": null,
     "seed": 60
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.76747077065221,
     "capability": 61.74359439203453,
     "intelligence": 39.71971801341685
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7026666666666667,
      "calibration": 87.99052671335875,
      "cc": 61.97531332000794,
      "cc_by_type": {
       "choice": 61.97531332000794
      },
      "cc_tiered": 61.41357724585651,
      "cc_tiered_open": 62.694715037494376,
      "cc_tiered_sealed": 60.13243945421864,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.97531332000794
     },
     "noul": {
      "accuracy": 0.6,
      "calibration": 78.80246798585758,
      "cc": 19.999999999999996,
      "cc_by_type": {
       "noul": 19.999999999999996
      },
      "cc_tiered": 17.242725862132826,
      "cc_tiered_open": 17.904071773636996,
      "cc_tiered_sealed": 16.581379950628655,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 19.999999999999996
     },
     "score": {
      "accuracy": 0.56,
      "calibration": 84.50941761274032,
      "cc": 39.16844975336833,
      "cc_by_type": {
       "score": 39.16844975336833
      },
      "cc_tiered": 40.502850932261246,
      "cc_tiered_open": 45.28625771631957,
      "cc_tiered_sealed": 35.719444148202925,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 39.16844975336833
     }
    },
    "raw_on_A": {
     "calibration": 81.01531873303975,
     "capability": 63.27790509971841,
     "intelligence": 45.54049146639707
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 36.515153858987674,
    "Emphasis on Accuracy 60:20:20": 30.874062935733846,
    "Emphasis on Cost 20:20:60": 38.11955193836121,
    "Emphasis on Speed 20:60:20": 42.49032904610869,
    "Intelligence only": 25.065620562020474
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Sqwish",
   "axes": {
    "calibration": 84.10309591585386,
    "cost": 62.69047197239138,
    "intelligence": 62.182714942969874,
    "speed": 94.89692360127322
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09316854395599922,
      "ece_score": 81.36629120880016,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 81.36629120880016,
      "score_before_time_limit": 81.36629120880016,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.13404825737265413,
      "brier": 0.09010097510472304,
      "ece": 0.0736929319861733,
      "n": 373,
      "score": 85.26141360276533,
      "score_before_time_limit": 85.26141360276533,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8736559139784946,
      "interval90_width": 0.24484767025089602,
      "n": 372,
      "nrps": 0.08489712327077574,
      "score": 85.68158293599608,
      "score_before_time_limit": 85.68158293599608,
      "time_limit_failed_share": 0,
      "top_ece": 0.10073560900465137
     }
    },
    "score": 84.10309591585386
   },
   "capability": 73.14290542941187,
   "composite_ci95": {
    "A": [
     70.52362840431199,
     74.9752910408981
    ],
    "B": [
     67.09508922559976,
     73.08094665903728
    ],
    "C": [
     61.9510288460444,
     74.9752910408981
    ]
   },
   "cost": {
    "basis": "Sqwish public price list, USD 0.04/M input, output not billed (docs.sqwish.ai/guides/pricing, fetched 16 Oct 2026 16:17Z, saved copy sha256 698a29e5...): kettei chub at USD 0.04 per million input tokens for decisions; \"The models generate no output tokens, and none are billed\"; \"standard list prices, the same since launch on 14 October 2026\". Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap, d1-ultra, Sage 1.4 and Nimble), through a scoped re-freeze of the frozen pricing module that adds the two kettei keys and nothing else, with a zero-difference proof for every other registry entry. The once-per-inbox USD 5.00 starter credit is a trial credit and is not used. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (650,605 input tokens over the 1,485 answered items, 0 output tokens), divided by the 1,485 items of the common cost basis the scorer uses; the 15 items refused with HTTP 422 count as zero tokens and as wrong answers, as every refused request on this draw does (d1-ultra's identical 15 refusals included). The unchanged runner also wrote a conservative cost ceiling into those 15 refused rows (65,536 input tokens each, the model's input limit); the scoring copy drops that one field on those 15 rows only. With the ceiling charged on all 1,500 items (the former all-items basis), the figure would be USD 0.043564 per 1,000 decisions (0.674x the official cost cap); that is a sensitivity, not this row's cost, and on the common basis the 15 refused items are not part of cost at all. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.017525 per 1,000 decisions (0.271x the official cost cap USD 0.064595), against USD 0.017349 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.017349466666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.017524713804713803
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kettei-chub (Sqwish)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.sqwish.ai/v1/systemone API, model `kettei-chub` (Sqwish docs: kettei-chub v1.0, released 14 Oct 2026), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `sqwish` before the first request (under the pre-registration). Native probabilities on all three question types. The runner recorded kettei-chub as the answering model on all 1,500 rows (the adapter reads the response's model field), so no row was answered by a fallback size. 15 of 1,500 items exceed the model's documented 65,536-token input limit (HTTP 422 context_too_long) and are scored as failures. paid evaluation, run with the customer's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 65.83625821719068,
    "I_sealed": 58.52917166874907,
    "base": 62.182714942969874,
    "excess": -2.532440221914108,
    "gap": 7.307086548441603,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 74.058241050527,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 62.25228059138094,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 71.38647342995169,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 70,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 52.064060171093324,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 59.46376557960574,
       "hard": 63.16197369506109,
       "judge": 50.34751163838864,
       "standard": 28.74320321795858
      }
     },
     "sealed|choice": {
      "cc": 74.30006625391061,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 88.56416772554002,
       "hard": 69.81612119810339,
       "judge": 69.14789422135163,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 54.66369142951747,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 41.81818181818182,
       "judge": 56.92307692307692,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 46.62375732281914,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 75.69949020400291,
       "hard": 37.65779302682666,
       "judge": 46.53750160134448,
       "standard": 50.147203056424175
      }
     }
    }
   },
   "jevbench_score": 73.44712821007478,
   "key": "kettei-chub",
   "last_measured_on": "2026-10-16",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.sqwish.ai/v1/systemone#kettei-chub",
   "noul_decisive": {
    "acc_among_decisive": 0.9380804953560371,
    "decisive": 323,
    "decisive_rate": 0.8659517426273459,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 9,
    "axes": {
     "calibration": 84.10309591585386,
     "cost": 62.69047197239138,
     "intelligence": 62.182714942969874,
     "speed": 95.09167642607092
    },
    "board_release": "v1.6.43",
    "capability": 73.14290542941187,
    "capability_api_board_rank": 16,
    "capability_open_board_rank": null,
    "capability_rank": 32,
    "composite_ci95": {
     "A": [
      70.55047342358355,
      75.00563282943165
     ],
     "B": [
      67.11452609813675,
      73.104006933403
     ],
     "C": [
      61.974600790042366,
      75.00563282943165
     ]
    },
    "jevbench_score": 73.47624549745721,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 11,
    "ranks": {
     "A": 11,
     "B": 16,
     "C": 11,
     "capability": 57
    },
    "scores": {
     "A": 73.47624549745721,
     "B": 70.90086398046398,
     "C": 73.47624549745721
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.16667446299106814,
     "p50_s_raw": 0.16667446299106814,
     "p95_s_adjusted": 0.18576474376022814,
     "p95_s_raw": 0.18576474376022814
    },
    "views": {
     "Balanced 33:33:33": 70.50662300412756,
     "Emphasis on Accuracy 60:20:20": 66.92322969828575,
     "Emphasis on Cost 20:20:60": 67.15739116260939,
     "Emphasis on Speed 20:60:20": 78.6391905348317,
     "Intelligence only": 62.182714942969874
    }
   },
   "rank": 40,
   "ranked": true,
   "ranks": {
    "A": 40,
    "B": 46,
    "C": 34,
    "capability": 75
   },
   "repo": "https://docs.sqwish.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.44712821007478,
    "B": 70.8791724952149,
    "C": 73.44712821007478
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.16808052101987414,
    "p50_s_raw": 0.16808052101987414,
    "p95_s_adjusted": 0.19265942253987306,
    "p95_s_raw": 0.19265942253987306
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 63.43823636827809,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 6.981375771500953
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 63.43823636827809
      },
      "calibration_gen": {
       "accuracy": 0.801980198019802,
       "cc": 63.075183513321825,
       "cc_by_type": {
        "choice": 87.27465535524921,
        "noul": 60.00000000000001,
        "score": 41.950895184716266
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 63.075183513321825
      },
      "composite_gen": {
       "accuracy": 0.9702970297029703,
       "cc": 94.94824383168736,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 100,
        "score": 88.09676401538727
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 94.94824383168736
      },
      "fairness": {
       "accuracy": 0.9166666666666666,
       "cc": 79.68499498178473,
       "cc_by_type": {
        "choice": 100,
        "noul": 46.66666666666666,
        "score": 92.38831827868752
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 79.68499498178473
      },
      "jaggedness": {
       "accuracy": 0.5888888888888889,
       "cc": 24.7828907621076,
       "cc_by_type": {
        "choice": 74.01347449470644,
        "noul": 8.108108108108114,
        "score": -7.772910316491766
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 24.7828907621076
      },
      "llm_router": {
       "accuracy": 0.8146964856230032,
       "cc": 71.82742478150287,
       "cc_by_type": {
        "choice": 68.3014880690323,
        "noul": 83.50515463917525,
        "score": 63.675631636301034
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 71.82742478150287
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5555555555555556,
       "cc": 23.877597944583083,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": 25,
        "score": 20.859597957460597
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 23.877597944583083
      },
      "multilingual": {
       "accuracy": 0.7709090909090909,
       "cc": 63.90048221318355,
       "cc_by_type": {
        "choice": 74.5679817411151,
        "noul": 61.111111111111114,
        "score": 56.02235378732444
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 63.90048221318355
      },
      "negation_pair_gen": {
       "accuracy": 0.872093023255814,
       "cc": 72.10157892455614,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 53.33333333333334,
        "score": 69.63807010700175
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 72.10157892455614
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.921875,
       "cc": 86.78313259496274,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 72.36140979690022
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 86.78313259496274
      },
      "usecase": {
       "accuracy": 0.6818181818181818,
       "cc": 55.999296711189196,
       "cc_by_type": {
        "choice": 64.4607843137255,
        "noul": 50,
        "score": 53.53710581984207
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 55.999296711189196
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 28.24640647098315,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": -8.117923444193398
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 28.24640647098315
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 71.36869026296806,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 100,
        "score": 55.58072516678434
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 71.36869026296806
      },
      "de": {
       "accuracy": 0.875,
       "cc": 83.2987215870243,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100,
        "score": 81.7545718407189
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.2987215870243
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 71.77688256637983,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 66.66666666666667,
        "score": 98.94331064141133
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 71.77688256637983
      },
      "en": {
       "accuracy": 0.7732406602953953,
       "cc": 61.55931282182781,
       "cc_by_type": {
        "choice": 75.39693193883403,
        "noul": 61.15107913669065,
        "score": 48.12992738995876
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 61.55931282182781
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 76.64478118166096,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 39.722271766842574
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 76.64478118166096
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 65.73563989131308,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 96.66047158650755
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.73563989131308
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 79.01802864795974,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 79.40702712034981
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 79.01802864795974
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 24.206837327473764,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 50,
        "score": -27.10015840864015
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 24.206837327473764
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 76.75527283143386,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 44.295436576920444
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 76.75527283143386
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 48.64168346881549,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 60.00000000000001,
        "score": 0.22211156609699412
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 48.64168346881549
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 49.09081184412445,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 47.27243553237336
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 49.09081184412445
      },
      "mixed": {
       "accuracy": 1,
       "cc": 98.50661742898735,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.51985228696206
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 98.50661742898735
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 60.007042228304385,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 98.63584529963177
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 60.007042228304385
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 61.31631791133893,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 51.566389197410786
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 61.31631791133893
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 63.091184704198746,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 61.09441445437582
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 63.091184704198746
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 54.828846215251076,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 55.55036843298726
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 54.828846215251076
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 56.63034829618355,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": -8.369824676666738
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 56.63034829618355
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 82.98189368154357,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 81.20374556075973
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 82.98189368154357
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 48.085566824177704,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 10.92336713919978
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 48.085566824177704
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8066666666666666,
       "cc": 75.02822480338315,
       "cc_by_type": {
        "choice": 75.02822480338315
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 75.02822480338315
      },
      "noul": {
       "accuracy": 0.808,
       "cc": 61.60000000000001,
       "cc_by_type": {
        "noul": 61.60000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 61.60000000000001
      },
      "score": {
       "accuracy": 0.672,
       "cc": 49.184305491330974,
       "cc_by_type": {
        "score": 49.184305491330974
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 49.184305491330974
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 32.94973949397677,
       "cc_by_type": {
        "choice": 36,
        "noul": 25,
        "score": 37.8492184819303
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 32.94973949397677
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 21.86713652267862,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 33.33333333333333,
        "score": -11.831302647285046
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 21.86713652267862
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 58.41468804345843,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 7.764389333627342
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 58.41468804345843
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 21.307121699711207,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": 24.155283227788605
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 21.307121699711207
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 52.732202808738386,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 53.84878233925865
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 52.732202808738386
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": -17.90003745614136,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 66.66666666666667,
        "score": -157.47369727408446
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 50.15463162953395,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 37.018516737341336
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 50.15463162953395
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 46.19866186338035,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 0,
        "score": 80.40783228003652
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 46.19866186338035
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 55.891544536440826,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 70.1746336093225
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 55.891544536440826
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 61.85269108394834,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 44.58733806269443
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 61.85269108394834
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 72.34106793586746,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 50.87333300656878
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 72.34106793586746
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 40.88339357227535,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 33.33333333333333,
        "score": 21.574911899621764
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 40.88339357227535
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 51.77378065698675,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 19.999999999999996,
        "score": 58.3539715295015
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 51.77378065698675
      },
      "model_routing": {
       "accuracy": 0.8847926267281107,
       "cc": 78.0080271816572,
       "cc_by_type": {
        "choice": 84.80519480519482,
        "noul": 96.96969696969697,
        "score": 52.24918977007981
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 78.0080271816572
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 41.95050174639673,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 0,
        "score": 60.40044957700209
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 41.95050174639673
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 56.20139697883294,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 68.69704144717662
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 56.20139697883294
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 36.63384674487239,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": 50,
        "score": 19.014840727228023
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 36.63384674487239
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 40.942225675125954,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 66.30493789494308
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 40.942225675125954
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.41000709626154,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 93.90617787241452
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.41000709626154
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.03674454380982,
      86.11285311974031
     ],
     "I": [
      56.070327300946786,
      66.51811397342259
     ],
     "capability": [
      68.26370889954131,
      75.68024782241592
     ],
     "composite": {
      "A": [
       70.52362840431199,
       74.9752910408981
      ],
      "B": [
       67.09508922559976,
       73.08094665903728
      ],
      "C": [
       61.9510288460444,
       74.9752910408981
      ]
     },
     "seed": 40
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.10309591585386,
     "capability": 73.14290542941187,
     "intelligence": 62.182714942969874
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8066666666666666,
      "calibration": 81.36629120880016,
      "cc": 75.02822480338315,
      "cc_by_type": {
       "choice": 75.02822480338315
      },
      "cc_tiered": 74.17915365221882,
      "cc_tiered_open": 74.058241050527,
      "cc_tiered_sealed": 74.30006625391061,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.02822480338315
     },
     "noul": {
      "accuracy": 0.808,
      "calibration": 85.26141360276533,
      "cc": 61.60000000000001,
      "cc_by_type": {
       "noul": 61.60000000000001
      },
      "cc_tiered": 63.02508242973458,
      "cc_tiered_open": 71.38647342995169,
      "cc_tiered_sealed": 54.66369142951747,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 61.60000000000001
     },
     "score": {
      "accuracy": 0.672,
      "calibration": 85.68158293599608,
      "cc": 49.184305491330974,
      "cc_by_type": {
       "score": 49.184305491330974
      },
      "cc_tiered": 49.34390874695623,
      "cc_tiered_open": 52.064060171093324,
      "cc_tiered_sealed": 46.62375732281914,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 49.184305491330974
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.47087864836172,
    "Emphasis on Accuracy 60:20:20": 66.90390345364955,
    "Emphasis on Cost 20:20:60": 67.1379294576571,
    "Emphasis on Speed 20:60:20": 78.55919303268641,
    "Intelligence only": 62.182714942969874
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Sqwish",
   "axes": {
    "calibration": 92.78078361709031,
    "cost": 57.40773420072094,
    "intelligence": 79.83918785475791,
    "speed": 92.90799647527135
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.024857318118089264,
      "ece_score": 95.02853637638215,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 95.02853637638215,
      "score_before_time_limit": 95.02853637638215,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.04557640750670244,
      "brier": 0.04418057925447822,
      "ece": 0.03819155325961758,
      "n": 373,
      "score": 92.36168934807648,
      "score_before_time_limit": 92.36168934807648,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9301075268817204,
      "interval90_width": 0.1270161290322581,
      "n": 372,
      "nrps": 0.044540233632597244,
      "score": 90.95212512681232,
      "score_before_time_limit": 90.95212512681232,
      "time_limit_failed_share": 0,
      "top_ece": 0.0682086319155783
     }
    },
    "score": 92.78078361709031
   },
   "capability": 86.3099857359241,
   "composite_ci95": {
    "A": [
     76.06572376300029,
     78.39596115963143
    ],
    "B": [
     75.94727293323967,
     79.2430710140646
    ],
    "C": [
     76.06572376300029,
     78.39596115963143
    ]
   },
   "cost": {
    "basis": "Sqwish public price list, USD 0.06/M input, output not billed (docs.sqwish.ai/guides/pricing, fetched 16 Oct 2026 16:17Z, saved copy sha256 698a29e5...): kettei fat at USD 0.06 per million input tokens for decisions; \"The models generate no output tokens, and none are billed\"; \"standard list prices, the same since launch on 14 October 2026\". Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap, d1-ultra, Sage 1.4 and Nimble), through a scoped re-freeze of the frozen pricing module that adds the two kettei keys and nothing else, with a zero-difference proof for every other registry entry. The once-per-inbox USD 5.00 starter credit is a trial credit and is not used. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (650,605 input tokens over the 1,485 answered items, 0 output tokens), divided by the 1,485 items of the common cost basis the scorer uses; the 15 items refused with HTTP 422 count as zero tokens and as wrong answers, as every refused request on this draw does (d1-ultra's identical 15 refusals included). The unchanged runner also wrote a conservative cost ceiling into those 15 refused rows (65,536 input tokens each, the model's input limit); the scoring copy drops that one field on those 15 rows only. With the ceiling charged on all 1,500 items (the former all-items basis), the figure would be USD 0.065346 per 1,000 decisions (1.012x the official cost cap); that is a sensitivity, not this row's cost, and on the common basis the 15 refused items are not part of cost at all. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.026287 per 1,000 decisions (0.407x the official cost cap USD 0.064595), against USD 0.026024 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.026024199999999994
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0262870707070707
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kettei-fat (Sqwish)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.sqwish.ai/v1/systemone API, model `kettei-fat` (Sqwish docs: kettei-fat v1.0, released 14 Oct 2026), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `sqwish` before the first request (under the pre-registration). Native probabilities on all three question types. The runner recorded kettei-fat as the answering model on all 1,500 rows (the adapter reads the response's model field), so no row was answered by a fallback size. 15 of 1,500 items exceed the model's documented 65,536-token input limit (HTTP 422 context_too_long) and are scored as failures. paid evaluation, run with the customer's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 79.51365708398285,
    "I_sealed": 80.16471862553297,
    "base": 79.83918785475791,
    "excess": -10.490588311905832,
    "gap": -0.6510615415501206,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 87.36347037521236,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 86.78829820698331,
       "judge": 88.13447593935399,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 80.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 71.10020619074588,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 82.05684261096903,
       "hard": 73.02709979951186,
       "judge": 72.7279902420528,
       "standard": 59.32642468614199
      }
     },
     "sealed|choice": {
      "cc": 87.64254301487159,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 100,
       "hard": 81.78559037816582,
       "judge": 89.42213516160628,
       "standard": 90.50833157561695
      }
     },
     "sealed|noul": {
      "cc": 83.10326832200224,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 93.10344827586208,
       "hard": 74.54545454545453,
       "judge": 93.84615384615384,
       "standard": 79.10447761194031
      }
     },
     "sealed|score": {
      "cc": 69.74834453972508,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 89.41045649485207,
       "hard": 62.86673539910686,
       "judge": 69.73247418036178,
       "standard": 73.70431238244299
      }
     }
    }
   },
   "jevbench_score": 77.69122909321317,
   "key": "kettei-fat",
   "last_measured_on": "2026-10-16",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.sqwish.ai/v1/systemone#kettei-fat",
   "noul_decisive": {
    "acc_among_decisive": 0.9634831460674157,
    "decisive": 356,
    "decisive_rate": 0.9544235924932976,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 1,
    "axes": {
     "calibration": 92.78078361709031,
     "cost": 57.40773420072094,
     "intelligence": 79.83918785475791,
     "speed": 93.19124027169539
    },
    "board_release": "v1.6.43",
    "capability": 86.3099857359241,
    "capability_api_board_rank": 1,
    "capability_open_board_rank": null,
    "capability_rank": 1,
    "composite_ci95": {
     "A": [
      76.11307381850703,
      78.44625770405383
     ],
     "B": [
      75.98503036462549,
      79.28417747037292
     ],
     "C": [
      76.11307381850703,
      78.44625770405383
     ]
    },
    "jevbench_score": 77.74062514641476,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 1,
    "ranks": {
     "A": 1,
     "B": 1,
     "C": 1,
     "capability": 6
    },
    "scores": {
     "A": 77.74062514641476,
     "B": 78.15146537474011,
     "C": 77.74062514641476
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.20005768549162894,
     "p50_s_raw": 0.20005768549162894,
     "p95_s_adjusted": 0.2397290877619525,
     "p95_s_raw": 0.2397290877619525
    },
    "views": {
     "Balanced 33:33:33": 73.7552774018824,
     "Emphasis on Accuracy 60:20:20": 76.0740779326146,
     "Emphasis on Cost 20:20:60": 66.21326749074146,
     "Emphasis on Speed 20:60:20": 80.46826083335421,
     "Intelligence only": 79.83918785475791
    }
   },
   "rank": 5,
   "ranked": true,
   "ranks": {
    "A": 5,
    "B": 2,
    "C": 4,
    "capability": 5
   },
   "repo": "https://docs.sqwish.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 77.69122909321317,
    "B": 78.11152478647084,
    "C": 77.69122909321317
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20237550800084136,
    "p50_s_raw": 0.20237550800084136,
    "p95_s_adjusted": 0.2529544946068199,
    "p95_s_raw": 0.2529544946068199
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 96.7507472305415,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 98.58557502495783
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 96.7507472305415
      },
      "calibration_gen": {
       "accuracy": 0.9603960396039604,
       "cc": 93.47881080044199,
       "cc_by_type": {
        "choice": 96.8186638388123,
        "noul": 92,
        "score": 91.6177685625137
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 93.47881080044199
      },
      "composite_gen": {
       "accuracy": 0.9900990099009901,
       "cc": 98.5804818840001,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.74144565200028
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 98.5804818840001
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 84.86062527550072,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 94.58187582650217
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.86062527550072
      },
      "jaggedness": {
       "accuracy": 0.6777777777777778,
       "cc": 48.399546734792864,
       "cc_by_type": {
        "choice": 74.01347449470644,
        "noul": 35.13513513513513,
        "score": 36.05003057453702
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 48.399546734792864
      },
      "llm_router": {
       "accuracy": 0.952076677316294,
       "cc": 87.82993917915225,
       "cc_by_type": {
        "choice": 96.83014880690322,
        "noul": 97.93814432989691,
        "score": 68.72152440065662
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 87.82993917915225
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.8518518518518519,
       "cc": 83.41157557420702,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 100,
        "score": 87.34812878447671
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 83.41157557420702
      },
      "multilingual": {
       "accuracy": 0.8618181818181818,
       "cc": 78.9798737677216,
       "cc_by_type": {
        "choice": 80.43690903162701,
        "noul": 86.11111111111111,
        "score": 70.39160116042665
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 78.9798737677216
      },
      "negation_pair_gen": {
       "accuracy": 0.9767441860465116,
       "cc": 94.85365625545666,
       "cc_by_type": {
        "choice": 100,
        "noul": 86.66666666666667,
        "score": 97.8943020997033
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 94.85365625545666
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.9375,
       "cc": 89.64749433680471,
       "cc_by_type": {
        "choice": 93.993993993994,
        "noul": 100,
        "score": 74.94848901642015
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 89.64749433680471
      },
      "usecase": {
       "accuracy": 0.8590909090909091,
       "cc": 79.73490795932031,
       "cc_by_type": {
        "choice": 88.97058823529412,
        "noul": 75,
        "score": 75.2341356426668
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 79.73490795932031
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 86.21024275740876,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": 77.67834731984534
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 86.21024275740876
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 75.04719104747973,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 45.87890033137929
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 75.04719104747973
      },
      "de": {
       "accuracy": 0.875,
       "cc": 86.628058606648,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100,
        "score": 91.74258289959003
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 86.628058606648
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.61393066873585,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.69467841216657
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.61393066873585
      },
      "en": {
       "accuracy": 0.8835794960903562,
       "cc": 80.4883103640632,
       "cc_by_type": {
        "choice": 89.67549822433213,
        "noul": 80.57553956834533,
        "score": 71.21389329951214
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 80.4883103640632
      },
      "es": {
       "accuracy": 0.967741935483871,
       "cc": 90.39349165083537,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 71.18047495250612
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 90.39349165083537
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.5139405386418,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.2084882825921
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.5139405386418
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 91.9301217185329,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 96.966835743834
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 91.9301217185329
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 79.08766846078485,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100,
        "score": 54.022781918667405
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 79.08766846078485
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 80.52064370654271,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 55.59154920224702
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 80.52064370654271
      },
      "ja": {
       "accuracy": 0.92,
       "cc": 80.92306844140934,
       "cc_by_type": {
        "choice": 100,
        "noul": 80,
        "score": 62.76920532422802
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 80.92306844140934
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 85.64585125280645,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 56.93755375841938
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 85.64585125280645
      },
      "mixed": {
       "accuracy": 1,
       "cc": 97.20181470806706,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 91.60544412420121
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 97.20181470806706
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 15.179397099416787,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 33.33333333333333,
        "score": -61.821116061057005
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 15.179397099416787
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8717948717948718,
       "cc": 86.0234580413718,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 100,
        "score": 73.30524505090337
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 86.0234580413718
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 75.9847623403091,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 59.7751473627069
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 75.9847623403091
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 67.67066900776906,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 100,
        "score": 54.07583681054125
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 67.67066900776906
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 83.57430186853024,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 72.46203604037333
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 83.57430186853024
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 81.78317833096776,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 77.60759950903231
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 81.78317833096776
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 46.85913519178663,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 7.244072242026545
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.85913519178663
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.9053333333333333,
       "cc": 87.77244111062208,
       "cc_by_type": {
        "choice": 87.77244111062208
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 87.77244111062208
      },
      "noul": {
       "accuracy": 0.9146666666666666,
       "cc": 82.93333333333332,
       "cc_by_type": {
        "noul": 82.93333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 82.93333333333332
      },
      "score": {
       "accuracy": 0.7973333333333333,
       "cc": 70.94010320323034,
       "cc_by_type": {
        "score": 70.94010320323034
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 70.94010320323034
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6818181818181818,
       "cc": 49.66614749506439,
       "cc_by_type": {
        "choice": 68,
        "noul": 50,
        "score": 30.998442485193188
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 49.66614749506439
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 91.94328546901535,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 75.82985640704605
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 91.94328546901535
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 51.2790489623809,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": -13.642527909605295
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 51.2790489623809
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 14.651774389394646,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 100,
        "score": -74.1733318025763
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 14.651774389394646
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.9411764705882353,
       "cc": 90.47904341608493,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 71.43713024825477
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 90.47904341608493
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 47.918819771310574,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 66.66666666666667,
        "score": -10.331591000533692
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 47.918819771310574
      },
      "financial_crime": {
       "accuracy": 0.8333333333333334,
       "cc": 77.48796585461322,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 71.42857142857142,
        "score": 61.03532613526828
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 77.48796585461322
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 66.73764536574002,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 50,
        "score": 71.11885944216777
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 66.73764536574002
      },
      "insurance_claims": {
       "accuracy": 0.8666666666666667,
       "cc": 77.28653895219419,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 60.00000000000001,
        "score": 92.69295018991592
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 77.28653895219419
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 75.55654372603391,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 72.85093025019309
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 75.55654372603391
      },
      "lead_generation": {
       "accuracy": 0.88,
       "cc": 87.19694978627531,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 100,
        "score": 73.21875633557016
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 87.19694978627531
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 52.35634073814115,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 88.25181791334818
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 52.35634073814115
      },
      "llm_guardrails": {
       "accuracy": 0.9259259259259259,
       "cc": 82.92727279464067,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 88.781818383922
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 82.92727279464067
      },
      "model_routing": {
       "accuracy": 0.9262672811059908,
       "cc": 84.3082332710354,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100,
        "score": 58.76885565726209
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 84.3082332710354
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 56.11137155389184,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0,
        "score": 79.85042944094617
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 56.11137155389184
      },
      "recruiting": {
       "accuracy": 0.8888888888888888,
       "cc": 89.83757082391922,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100,
        "score": 86.22580439376324
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 89.83757082391922
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 69.35928856610843,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 100,
        "score": 37.63451594463072
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 69.35928856610843
      },
      "scientific_discovery": {
       "accuracy": 0.8695652173913043,
       "cc": 84.81707822400728,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 100,
        "score": 83.43674191839865
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 84.81707822400728
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.38110705803847,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.14332117411539
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.38110705803847
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.77958369550683,
      93.35538700668452
     ],
     "I": [
      75.36869158042848,
      83.13854440546274
     ],
     "capability": [
      82.60221375083708,
      87.8846855518411
     ],
     "composite": {
      "A": [
       76.06572376300029,
       78.39596115963143
      ],
      "B": [
       75.94727293323967,
       79.2430710140646
      ],
      "C": [
       76.06572376300029,
       78.39596115963143
      ]
     },
     "seed": 41
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 92.78078361709031,
     "capability": 86.3099857359241,
     "intelligence": 79.83918785475791
    },
    "per_type": {
     "choice": {
      "accuracy": 0.9053333333333333,
      "calibration": 95.02853637638215,
      "cc": 87.77244111062208,
      "cc_by_type": {
       "choice": 87.77244111062208
      },
      "cc_tiered": 87.50300669504198,
      "cc_tiered_open": 87.36347037521236,
      "cc_tiered_sealed": 87.64254301487159,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 87.77244111062208
     },
     "noul": {
      "accuracy": 0.9146666666666666,
      "calibration": 92.36168934807648,
      "cc": 82.93333333333332,
      "cc_by_type": {
       "noul": 82.93333333333332
      },
      "cc_tiered": 81.5902815039963,
      "cc_tiered_open": 80.07729468599035,
      "cc_tiered_sealed": 83.10326832200224,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 82.93333333333332
     },
     "score": {
      "accuracy": 0.7973333333333333,
      "calibration": 90.95212512681232,
      "cc": 70.94010320323034,
      "cc_by_type": {
       "score": 70.94010320323034
      },
      "cc_tiered": 70.42427536523547,
      "cc_tiered_open": 71.10020619074588,
      "cc_tiered_sealed": 69.74834453972508,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 70.94010320323034
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 73.69600561562211,
    "Emphasis on Accuracy 60:20:20": 76.03623197488143,
    "Emphasis on Cost 20:20:60": 66.184595083788,
    "Emphasis on Speed 20:60:20": 80.34136502055784,
    "Intelligence only": 79.83918785475791
   }
  },
  {
   "adapter_id": "images/kev-0.5b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jared Palmer",
   "axes": {
    "calibration": 61.20618525995795,
    "cost": 84.19572866753369,
    "intelligence": 4.598433868646836,
    "speed": 92.39061085463217
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.22209098434273397,
      "ece_score": 55.5818031314532,
      "mean_tvd": null,
      "n": 729,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 55.5818031314532,
      "score_before_time_limit": 55.5818031314532,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.7439353099730458,
      "brier": 0.291543126684636,
      "ece": 0.1912938005390836,
      "n": 371,
      "score": 61.74123989218327,
      "score_before_time_limit": 61.74123989218327,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9239130434782609,
      "interval90_width": 0.835824275362319,
      "n": 368,
      "nrps": 0.21140059009020898,
      "score": 66.29551275623737,
      "score_before_time_limit": 66.29551275623737,
      "time_limit_failed_share": 0,
      "top_ece": 0.23134457739252184
     }
    },
    "score": 61.20618525995795
   },
   "capability": 32.90230956430239,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.5B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 748,376 input + 155,176 output tokens over 1,624 decisions = 556.374384236 tokens/decision. GPU-time cost USD 0.00336363640628665/1,000; old list-price cost USD 0.00460822660098522/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00336363640628665/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0033636364062866533
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kev 0.5B",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 4.849742544692461,
    "I_sealed": 4.3471251926012116,
    "base": 4.598433868646836,
    "excess": -2.0707702903469944,
    "gap": 0.5026173520912494,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 14.549227634077383,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": 11.292859389745214,
       "judge": 24.851680949241928,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -82.6086956521739,
       "judge": -100,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 17.000794018795805,
       "hard": -8.594117745107855,
       "judge": -32.75818090283609,
       "standard": -52.88804065591748
      }
     },
     "sealed|choice": {
      "cc": 13.041375577803636,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 17.522725593082576,
       "hard": 7.352941176470571,
       "judge": 18.06331471135942,
       "standard": 14.644660672496622
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -48,
       "hard": -78.64077669902912,
       "judge": -71.05263157894737,
       "standard": -66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 4.208590362582387,
       "hard": -22.033232980160843,
       "judge": -8.39299678470844,
       "standard": -23.154804312918653
      }
     }
    }
   },
   "jevbench_score": 0.13190015086675236,
   "key": "kev-0.5b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5578947368421052,
    "decisive_rate": 0.2560646900269542,
    "supported": true,
    "valid": 371
   },
   "open": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 61.20618525995795,
     "cost": 80.09398520305423,
     "intelligence": 4.598433868646836,
     "speed": 92.39904218127765
    },
    "board_release": "v1.6.43",
    "capability": 32.90230956430239,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 125,
    "capability_rank": 154,
    "composite_ci95": {
     "A": [
      0.024058315307534144,
      0.35808460603860887
     ],
     "B": [
      0.015784497925561473,
      0.2496641339569836
     ],
     "C": [
      0.016707163408009824,
      0.2486698653045895
     ]
    },
    "jevbench_score": 0.131588623265787,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 176,
    "rank": 220,
    "ranks": {
     "A": 220,
     "B": 220,
     "C": 220,
     "capability": 221
    },
    "scores": {
     "A": 0.131588623265787,
     "B": 0.08911346438551963,
     "C": 0.09138098837901876
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2379663866944611,
     "p50_s_raw": 0.043983193347230554,
     "p95_s_adjusted": 0.24186897580511868,
     "p95_s_raw": 0.04593448790255934
    },
    "views": {
     "Balanced 33:33:33": 0.10538840072414271,
     "Emphasis on Accuracy 60:20:20": 0.06258833605506466,
     "Emphasis on Cost 20:20:60": 0.15914261116628434,
     "Emphasis on Speed 20:60:20": 0.1611592968234279,
     "Intelligence only": 0.03889464633042005
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 261,
   "ranked": true,
   "ranks": {
    "A": 261,
    "B": 261,
    "C": 261,
    "capability": 262
   },
   "repo": "https://github.com/jaredpalmer/kev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.13190015086675236,
    "B": 0.08922763796221615,
    "C": 0.09159732699080027
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2380974093452096,
    "p50_s_raw": 0.0440487046726048,
    "p95_s_adjusted": 0.24220563587732613,
    "p95_s_raw": 0.04610281793866307
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen2.5-0.5B + LoRA + learned pointer head; jaredpalmer/kev-0.5b",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      55.41312884823437,
      65.28644013850979
     ],
     "I": [
      2.5515112413087477,
      6.5563351690373795
     ],
     "capability": [
      29.53128001342342,
      35.43777813183522
     ],
     "composite": null,
     "seed": 52
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 61.20618525995795,
     "capability": 32.90230956430239,
     "intelligence": 4.598433868646836
    },
    "per_type": {
     "choice": {
      "accuracy": 0.31866666666666665,
      "calibration": 55.5818031314532,
      "cc": 12.8671977871034,
      "cc_by_type": {
       "choice": 12.8671977871034
      },
      "cc_tiered": 13.795301605940509,
      "cc_tiered_open": 14.549227634077383,
      "cc_tiered_sealed": 13.041375577803636,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 12.8671977871034
     },
     "noul": {
      "accuracy": 0.14133333333333334,
      "calibration": 61.74123989218327,
      "cc": -71.73333333333333,
      "cc_by_type": {
       "noul": -71.73333333333333
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.272,
      "calibration": 66.29551275623737,
      "cc": -15.865897379211603,
      "cc_by_type": {
       "score": -15.865897379211603
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 65.8831405050141,
     "capability": 35.92446865234692,
     "intelligence": 5.965796799679742
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.028000000000000025,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.10565487419772726,
    "Emphasis on Accuracy 60:20:20": 0.0626446349889732,
    "Emphasis on Cost 20:20:60": 0.16024232194623098,
    "Emphasis on Speed 20:60:20": 0.16153187851699238,
    "Intelligence only": 0.03889464633042005
   }
  },
  {
   "adapter_id": "images/kev-0.6b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jared Palmer",
   "axes": {
    "calibration": 66.26655558235618,
    "cost": 84.20067552024135,
    "intelligence": 7.598789169415934,
    "speed": 91.48734752715623
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.20778793892461145,
      "ece_score": 58.44241221507771,
      "mean_tvd": null,
      "n": 731,
      "n_ece_hard": 306,
      "n_tvd": 0,
      "score": 58.44241221507771,
      "score_before_time_limit": 58.44241221507771,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.568733153638814,
      "brier": 0.2628086253369272,
      "ece": 0.18765498652291104,
      "n": 371,
      "score": 62.4690026954178,
      "score_before_time_limit": 62.4690026954178,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9320652173913043,
      "interval90_width": 0.8045742753623186,
      "n": 368,
      "nrps": 0.17410404691603434,
      "score": 77.88825183657305,
      "score_before_time_limit": 77.88825183657305,
      "time_limit_failed_share": 0,
      "top_ece": 0.1340654581762524
     }
    },
    "score": 66.26655558235618
   },
   "capability": 36.93267237588606,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.6B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 748,375 input + 154,834 output tokens over 1,624 decisions = 556.16317734 tokens/decision. GPU-time cost USD 0.00336235952649738/1,000; old list-price cost USD 0.00460822044334975/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00336235952649738/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003362359526497381
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kev 0.6B (research preview)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 8.945068603011505,
    "I_sealed": 6.252509735820362,
    "base": 7.598789169415934,
    "excess": 0.1191712247528991,
    "gap": 2.692558867191143,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 16.179258482292497,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 45.513513513513516,
       "hard": 9.405473419314262,
       "judge": 24.851680949241928,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0,
       "hard": -73.91304347826086,
       "judge": -60,
       "standard": -44.44444444444444
      }
     },
     "open|score": {
      "cc": 10.655947326742021,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 23.83763830928548,
       "hard": -1.1034131190897023,
       "judge": 28.04844552251834,
       "standard": 1.4950754334692617
      }
     },
     "sealed|choice": {
      "cc": 18.757529207461086,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 34.8163476461459,
       "hard": 6.334841628959255,
       "judge": 24.581005586592198,
       "standard": 26.838280576425678
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -4.0000000000000036,
       "hard": -66.99029126213591,
       "judge": -44.73684210526315,
       "standard": -15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 11.09665079983172,
       "hard": -14.21346816054152,
       "judge": 12.732793638645301,
       "standard": -1.8098672811444283
      }
     }
    }
   },
   "jevbench_score": 0.5450619596118789,
   "key": "kev-0.6b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.69375,
    "decisive_rate": 0.431266846361186,
    "supported": true,
    "valid": 371
   },
   "open": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 66.26655558235618,
     "cost": 80.09400261254245,
     "intelligence": 7.598789169415934,
     "speed": 91.51497362122684
    },
    "board_release": "v1.6.43",
    "capability": 36.93267237588606,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 120,
    "capability_rank": 148,
    "composite_ci95": {
     "A": [
      0.13850323393885078,
      1.4591062434238613
     ],
     "B": [
      0.09387316749385785,
      1.070943045752662
     ],
     "C": [
      0.09618280134642418,
      1.0132682245999038
     ]
    },
    "jevbench_score": 0.5431213026023877,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 160,
    "rank": 203,
    "ranks": {
     "A": 203,
     "B": 203,
     "C": 203,
     "capability": 212
    },
    "scores": {
     "A": 0.5431213026023877,
     "B": 0.382771242886796,
     "C": 0.377167571251658
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26348884413018825,
     "p50_s_raw": 0.05674442206509411,
     "p95_s_adjusted": 0.2677567361854017,
     "p95_s_raw": 0.05887836809270084
    },
    "views": {
     "Balanced 33:33:33": 0.4469959095569562,
     "Emphasis on Accuracy 60:20:20": 0.2761353719898703,
     "Emphasis on Cost 20:20:60": 0.6416336282227073,
     "Emphasis on Speed 20:60:20": 0.6529389194958221,
     "Intelligence only": 0.17550648828077675
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 245,
   "ranked": true,
   "ranks": {
    "A": 245,
    "B": 245,
    "C": 245,
    "capability": 253
   },
   "repo": "https://github.com/jaredpalmer/kev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.5450619596118789,
    "B": 0.38354116598373367,
    "C": 0.3785152497304713
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26317979032173755,
    "p50_s_raw": 0.056589895160868764,
    "p95_s_adjusted": 0.2697818389628083,
    "p95_s_raw": 0.059890919481404126
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-0.6B-Base + LoRA + learned pointer head; jaredpalmer/kev-0.6b",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      60.54900964119034,
      70.35051467735538
     ],
     "I": [
      4.680395319947538,
      10.897085922837281
     ],
     "capability": [
      33.22958246000976,
      39.72314225130261
     ],
     "composite": null,
     "seed": 53
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 66.26655558235618,
     "capability": 36.93267237588606,
     "intelligence": 7.598789169415934
    },
    "per_type": {
     "choice": {
      "accuracy": 0.368,
      "calibration": 58.44241221507771,
      "cc": 19.176226518761275,
      "cc_by_type": {
       "choice": 19.176226518761275
      },
      "cc_tiered": 17.468393844876793,
      "cc_tiered_open": 16.179258482292497,
      "cc_tiered_sealed": 18.757529207461086,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 19.176226518761275
     },
     "noul": {
      "accuracy": 0.296,
      "calibration": 62.4690026954178,
      "cc": -40.800000000000004,
      "cc_by_type": {
       "noul": -40.800000000000004
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.30933333333333335,
      "calibration": 77.88825183657305,
      "cc": -0.08941736278893497,
      "cc_by_type": {
       "score": -0.08941736278893497
      },
      "cc_tiered": 5.327973663371011,
      "cc_tiered_open": 10.655947326742021,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 65.17717796738081,
     "capability": 37.13879390532707,
     "intelligence": 9.100409843273326
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.02533333333333332,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.4487491926320956,
    "Emphasis on Accuracy 60:20:20": 0.27653584156898564,
    "Emphasis on Cost 20:20:60": 0.6482009553939799,
    "Emphasis on Speed 20:60:20": 0.6551579149377957,
    "Intelligence only": 0.17550648828077675
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jared Palmer",
   "axes": {
    "calibration": 93.71720359195915,
    "cost": 56.1644780115139,
    "intelligence": 64.16071886717361,
    "speed": 82.8278891625431
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.034987296416937964,
      "ece_score": 93.0025407166124,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 93.0025407166124,
      "score_before_time_limit": 93.0025407166124,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.20485175202156336,
      "brier": 0.06491574711590294,
      "ece": 0.03041293800539084,
      "n": 371,
      "score": 93.91741239892183,
      "score_before_time_limit": 93.91741239892183,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.986449864498645,
      "interval90_width": 0.3089430894308942,
      "n": 369,
      "nrps": 0.049457903703703704,
      "score": 94.23165766034325,
      "score_before_time_limit": 94.23165766034325,
      "time_limit_failed_share": 0,
      "top_ece": 0.03295447154471563
     }
    },
    "score": 93.71720359195915
   },
   "capability": 78.93896122956639,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 663,658 input + 167,418 output tokens over 1,477 decisions = 562.678402167 tokens/decision. GPU-time cost USD 0.0289190529390902/1,000; old list-price cost USD 0.528768016249154/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0289190529390902/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.52066024
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.028919052939090185
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Kev 27B (Kev 1.0, Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 64.07452328888938,
    "I_sealed": 64.24691444545786,
    "base": 64.16071886717361,
    "excess": -2.745778799006724,
    "gap": -0.17239115656848014,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 81.24201973326763,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 79.23875432525952,
       "judge": 80.22412656558997,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 47.325051759834366,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 63.656498373566166,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.13878787878788,
       "hard": 65.75611111111111,
       "judge": 63.57169491525423,
       "standard": 57.843333333333334
      }
     },
     "sealed|choice": {
      "cc": 78.44659446797311,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 73.52941176470587,
       "judge": 80.44692737430168,
       "standard": 84.48084739499939
      }
     },
     "sealed|noul": {
      "cc": 50.836710400368496,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72,
       "hard": 33.98058252427185,
       "judge": 57.89473684210527,
       "standard": 63.38028169014085
      }
     },
     "sealed|score": {
      "cc": 63.45743846803195,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 71.6254609929078,
       "hard": 54.08627450980392,
       "judge": 71.25748502994011,
       "standard": 66.4156852791878
      }
     }
    }
   },
   "jevbench_score": 71.25763456915091,
   "key": "kev-27b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "jaredpalmer/kev-27b@af0e6d551bdc2cc724f3e9d7a8bee1cd4fb8f7bf",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9694915254237289,
    "decisive_rate": 0.7951482479784366,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 93.71720359195915,
     "cost": 18.302044660887503,
     "intelligence": 64.16071886717361,
     "speed": 83.65684843680415
    },
    "board_release": "v1.6.43",
    "capability": 78.93896122956639,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      5.665703330300189,
      5.813398804963979
     ],
     "B": [
      6.009217533619645,
      6.271823804033908
     ],
     "C": [
      5.484111432603801,
      5.813398804963979
     ]
    },
    "jevbench_score": 5.77223551264444,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 144,
    "rank": 184,
    "ranks": {
     "A": 184,
     "B": 180,
     "C": 178,
     "capability": 29
    },
    "scores": {
     "A": 5.77223551264444,
     "B": 6.178201442095797,
     "C": 5.77223551264444
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5102782679954544,
     "p50_s_raw": 0.18013913399772719,
     "p95_s_adjusted": 0.8443219571985537,
     "p95_s_raw": 0.34716097859927686
    },
    "views": {
     "Balanced 33:33:33": 4.891297337690537,
     "Emphasis on Accuracy 60:20:20": 5.910281375617489,
     "Emphasis on Cost 20:20:60": 3.499139205379304,
     "Emphasis on Speed 20:60:20": 6.315006752900174,
     "Intelligence only": 8.596633940261478
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 66,
   "ranked": true,
   "ranks": {
    "A": 66,
    "B": 57,
    "C": 52,
    "capability": 39
   },
   "repo": "https://huggingface.co/jaredpalmer/kev-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.25763456915091,
    "B": 69.71536973521941,
    "C": 71.25763456915091
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.524460306018591,
    "p50_s_raw": 0.18723015300929546,
    "p95_s_adjusted": 0.9942565476871092,
    "p95_s_raw": 0.4221282738435546
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.38693398193061,
      92.90424439872974
     ],
     "I": [
      58.981544658104795,
      68.3141526180369
     ],
     "capability": [
      74.70682599002546,
      79.91812944044146
     ],
     "composite": null,
     "seed": 62
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 93.71720359195915,
     "capability": 78.93896122956639,
     "intelligence": 64.16071886717361
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8386666666666667,
      "calibration": 93.0025407166124,
      "cc": 79.36777090457831,
      "cc_by_type": {
       "choice": 79.36777090457831
      },
      "cc_tiered": 79.84430710062037,
      "cc_tiered_open": 81.24201973326763,
      "cc_tiered_sealed": 78.44659446797311,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 79.36777090457831
     },
     "noul": {
      "accuracy": 0.7626666666666667,
      "calibration": 93.91741239892183,
      "cc": 52.533333333333346,
      "cc_by_type": {
       "noul": 52.533333333333346
      },
      "cc_tiered": 49.080881080101435,
      "cc_tiered_open": 47.325051759834366,
      "cc_tiered_sealed": 50.836710400368496,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 52.533333333333346
     },
     "score": {
      "accuracy": 0.704,
      "calibration": 94.23165766034325,
      "cc": 62.96492982456141,
      "cc_by_type": {
       "score": 62.96492982456141
      },
      "cc_tiered": 63.55696842079906,
      "cc_tiered_open": 63.656498373566166,
      "cc_tiered_sealed": 63.45743846803195,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.96492982456141
     }
    },
    "raw_on_A": {
     "calibration": 89.47604418949679,
     "capability": 78.40559149548258,
     "intelligence": 67.33513880146839
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.98636701217534,
    "Emphasis on Accuracy 60:20:20": 65.2437806495608,
    "Emphasis on Cost 20:20:60": 61.672332080805205,
    "Emphasis on Speed 20:60:20": 71.82835498893883,
    "Intelligence only": 64.16071886717361
   }
  },
  {
   "adapter_id": "images/kev-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jared Palmer",
   "axes": {
    "calibration": 68.45285936403559,
    "cost": 77.89103360168605,
    "intelligence": 19.510707720813777,
    "speed": 90.22194613896767
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.18711800351703844,
      "ece_score": 62.576399296592314,
      "mean_tvd": null,
      "n": 733,
      "n_ece_hard": 306,
      "n_tvd": 0,
      "score": 62.576399296592314,
      "score_before_time_limit": 62.576399296592314,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.32884097035040427,
      "brier": 0.19162911051212952,
      "ece": 0.13239892183288407,
      "n": 371,
      "score": 73.52021563342319,
      "score_before_time_limit": 73.52021563342319,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8365122615803815,
      "interval90_width": 0.4604904632152589,
      "n": 367,
      "nrps": 0.15784808091493369,
      "score": 69.26196316209126,
      "score_before_time_limit": 69.26196316209126,
      "time_limit_failed_share": 0,
      "top_ece": 0.22845632792162052
     }
    },
    "score": 68.45285936403559
   },
   "capability": 43.98178354242468,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 748,375 input + 153,531 output tokens over 1,624 decisions = 555.360837438 tokens/decision. GPU-time cost USD 0.00545714566821745/1,000; old list-price cost USD 0.0138246613300493/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00545714566821745/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005457145668217454
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kev 4B (research preview)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 19.939346418532395,
    "I_sealed": 19.08206902309516,
    "base": 19.510707720813777,
    "excess": -1.7161102470010072,
    "gap": 0.8572773954372366,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 51.235731456250235,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 45.26580685750237,
       "judge": 44.62755438365195,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": -4.347826086956519,
       "judge": -19.999999999999996,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 8.582307799346953,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 34.24426078971534,
       "hard": 15.014273649587162,
       "judge": 21.314617902468203,
       "standard": -36.211065550999535
      }
     },
     "sealed|choice": {
      "cc": 42.547775845492595,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 56.10080555760846,
       "hard": 29.751131221719444,
       "judge": 44.134078212290504,
       "standard": 58.9850966867841
      }
     },
     "sealed|noul": {
      "cc": 5.208014566705289,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -30.097087378640776,
       "judge": 26.315789473684205,
       "standard": 26.7605633802817
      }
     },
     "sealed|score": {
      "cc": 9.490416657087591,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 60.8431474353109,
       "hard": -5.464015519199061,
       "judge": 4.345254884769856,
       "standard": 21.440658279025847
      }
     }
    }
   },
   "jevbench_score": 6.78365205932627,
   "key": "kev-4b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8152610441767069,
    "decisive_rate": 0.6711590296495957,
    "supported": true,
    "valid": 371
   },
   "open": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 68.45285936403559,
     "cost": 65.78036497095258,
     "intelligence": 19.510707720813777,
     "speed": 90.46585183268016
    },
    "board_release": "v1.6.43",
    "capability": 43.98178354242468,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 103,
    "capability_rank": 131,
    "composite_ci95": {
     "A": [
      4.075524448580174,
      12.02725721176505
     ],
     "B": [
      3.1848561369918413,
      10.0344884794078
     ],
     "C": [
      2.83022531151401,
      8.352261952614619
     ]
    },
    "jevbench_score": 6.611791819903415,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 142,
    "rank": 182,
    "ranks": {
     "A": 182,
     "B": 184,
     "C": 183,
     "capability": 192
    },
    "scores": {
     "A": 6.611791819903415,
     "B": 5.310194944940352,
     "C": 4.591522097155148
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.29399301540106537,
     "p50_s_raw": 0.07199650770053267,
     "p95_s_adjusted": 0.3055468452628701,
     "p95_s_raw": 0.07777342263143508
    },
    "views": {
     "Balanced 33:33:33": 5.893456455385603,
     "Emphasis on Accuracy 60:20:20": 4.229224928104588,
     "Emphasis on Cost 20:20:60": 7.055012442959745,
     "Emphasis on Speed 20:60:20": 7.642572180991565,
     "Intelligence only": 2.970838616432075
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 226,
   "ranked": true,
   "ranks": {
    "A": 226,
    "B": 226,
    "C": 226,
    "capability": 231
   },
   "repo": "https://github.com/jaredpalmer/kev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 6.78365205932627,
    "B": 5.398063013273237,
    "C": 4.710869485643243
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.29449559943750503,
    "p50_s_raw": 0.0722477997187525,
    "p95_s_adjusted": 0.3226462163031101,
    "p95_s_raw": 0.08632310815155504
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-4B-Base + LoRA + learned pointer head; jaredpalmer/kev-4b",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      62.30814638480004,
      71.54002800358278
     ],
     "I": [
      16.228114704223106,
      24.71673328753193
     ],
     "capability": [
      39.998560456744215,
      47.239203316922335
     ],
     "composite": null,
     "seed": 54
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 68.45285936403559,
     "capability": 43.98178354242468,
     "intelligence": 19.510707720813777
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5693333333333334,
      "calibration": 62.576399296592314,
      "cc": 44.923884315527204,
      "cc_by_type": {
       "choice": 44.923884315527204
      },
      "cc_tiered": 46.891753650871415,
      "cc_tiered_open": 51.235731456250235,
      "cc_tiered_sealed": 42.547775845492595,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 44.923884315527204
     },
     "noul": {
      "accuracy": 0.5413333333333333,
      "calibration": 73.52021563342319,
      "cc": 8.266666666666666,
      "cc_by_type": {
       "noul": 8.266666666666666
      },
      "cc_tiered": 2.6040072833526446,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 5.208014566705289,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 8.266666666666666
     },
     "score": {
      "accuracy": 0.41333333333333333,
      "calibration": 69.26196316209126,
      "cc": 12.570612763030764,
      "cc_by_type": {
       "score": 12.570612763030764
      },
      "cc_tiered": 9.036362228217271,
      "cc_tiered_open": 8.582307799346953,
      "cc_tiered_sealed": 9.490416657087591,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 12.570612763030764
     }
    },
    "raw_on_A": {
     "calibration": 69.97372951951687,
     "capability": 47.498407356877294,
     "intelligence": 25.023085194237723
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.022666666666666613,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.021333333333333315,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 6.076413359498034,
    "Emphasis on Accuracy 60:20:20": 4.284773295111892,
    "Emphasis on Cost 20:20:60": 7.5489594713818065,
    "Emphasis on Speed 20:60:20": 7.821107285603946,
    "Intelligence only": 2.970838616432075
   }
  },
  {
   "adapter_id": "images/kev-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Jared Palmer",
   "axes": {
    "calibration": 70.7561729087212,
    "cost": 68.86568213459405,
    "intelligence": 29.26217941280341,
    "speed": 88.45850101024836
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14254951638954755,
      "ece_score": 71.49009672209048,
      "mean_tvd": null,
      "n": 736,
      "n_ece_hard": 306,
      "n_tvd": 0,
      "score": 71.49009672209048,
      "score_before_time_limit": 71.49009672209048,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.23450134770889486,
      "brier": 0.18970835579514836,
      "ece": 0.16199460916442043,
      "n": 371,
      "score": 67.60107816711592,
      "score_before_time_limit": 67.60107816711592,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8233695652173914,
      "interval90_width": 0.382925724637681,
      "n": 368,
      "nrps": 0.13575035733208884,
      "score": 73.17734383695722,
      "score_before_time_limit": 73.17734383695722,
      "time_limit_failed_share": 0,
      "top_ece": 0.20035138296438337
     }
    },
    "score": 70.7561729087212
   },
   "capability": 50.0091761607623,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 8B active parameters; throughput 30762.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 748,375 input + 153,147 output tokens over 1,624 decisions = 555.124384236 tokens/decision. GPU-time cost USD 0.0109096444132819/1,000; old list-price cost USD 0.0968237438423645/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0109096444132819/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01090964441328195
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "kev 8B (research preview)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 28.412661059008222,
    "I_sealed": 30.111697766598596,
    "base": 29.26217941280341,
    "excess": -4.272424350028617,
    "gap": -1.6990367075903734,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 53.56342531883604,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 43.378420887071414,
       "judge": 48.58272907053395,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 15.995169082125601,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 13.043478260869556,
       "judge": 10.000000000000009,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 15.679388776063027,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 43.66574839302112,
       "hard": 33.21276294296097,
       "judge": 29.41280568734839,
       "standard": -53.98066473313996
      }
     },
     "sealed|choice": {
      "cc": 48.755934054679,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 61.4219200354741,
       "hard": 41.96832579185519,
       "judge": 49.72067039106147,
       "standard": 54.55105308535534
      }
     },
     "sealed|noul": {
      "cc": 19.06361274442773,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": -18.446601941747577,
       "judge": 50,
       "standard": 35.2112676056338
      }
     },
     "sealed|score": {
      "cc": 22.51554650068906,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 58.4626547761159,
       "hard": 14.194388556502712,
       "judge": 23.600291166841224,
       "standard": 19.557191252120088
      }
     }
    }
   },
   "jevbench_score": 18.48094036640983,
   "key": "kev-8b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8063380281690141,
    "decisive_rate": 0.7654986522911051,
    "supported": true,
    "valid": 371
   },
   "open": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 70.7561729087212,
     "cost": 40.42054386073457,
     "intelligence": 29.26217941280341,
     "speed": 88.97143473610706
    },
    "board_release": "v1.6.43",
    "capability": 50.0091761607623,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      6.289030902475565,
      16.488978582767544
     ],
     "B": [
      5.395378045720429,
      15.122131514496889
     ],
     "C": [
      4.367382571163586,
      11.45067957136635
     ]
    },
    "jevbench_score": 10.62283914209564,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 130,
    "rank": 169,
    "ranks": {
     "A": 169,
     "B": 169,
     "C": 169,
     "capability": 173
    },
    "scores": {
     "A": 10.62283914209564,
     "B": 9.44790466894511,
     "C": 7.3769716264553065
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2989678788930178,
     "p50_s_raw": 0.07448393944650888,
     "p95_s_adjusted": 0.42386933271773153,
     "p95_s_raw": 0.13693466635886578
    },
    "views": {
     "Balanced 33:33:33": 9.572183421009896,
     "Emphasis on Accuracy 60:20:20": 8.080814281596565,
     "Emphasis on Cost 20:20:60": 9.355272676593998,
     "Emphasis on Speed 20:60:20": 12.08216537216834,
     "Intelligence only": 6.550045158904166
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 190,
   "ranked": true,
   "ranks": {
    "A": 190,
    "B": 191,
    "C": 191,
    "capability": 211
   },
   "repo": "https://github.com/jaredpalmer/kev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 18.48094036640983,
    "B": 15.81208641065244,
    "C": 12.833986365562383
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3028102615848184,
    "p50_s_raw": 0.07640513079240918,
    "p95_s_adjusted": 0.47095488999038937,
    "p95_s_raw": 0.16047744499519467
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-8B-Base + LoRA + learned pointer head; jaredpalmer/kev-8b",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      65.36869404076147,
      74.24437950494625
     ],
     "I": [
      23.662522876918093,
      35.13692561855455
     ],
     "capability": [
      45.238684166857645,
      53.83400001933101
     ],
     "composite": null,
     "seed": 55
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 70.7561729087212,
     "capability": 50.0091761607623,
     "intelligence": 29.26217941280341
    },
    "per_type": {
     "choice": {
      "accuracy": 0.612,
      "calibration": 71.49009672209048,
      "cc": 50.38034159696103,
      "cc_by_type": {
       "choice": 50.38034159696103
      },
      "cc_tiered": 51.15967968675752,
      "cc_tiered_open": 53.56342531883604,
      "cc_tiered_sealed": 48.755934054679,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 50.38034159696103
     },
     "noul": {
      "accuracy": 0.6106666666666667,
      "calibration": 67.60107816711592,
      "cc": 22.13333333333334,
      "cc_by_type": {
       "noul": 22.13333333333334
      },
      "cc_tiered": 17.529390913276664,
      "cc_tiered_open": 15.995169082125601,
      "cc_tiered_sealed": 19.06361274442773,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 22.13333333333334
     },
     "score": {
      "accuracy": 0.49866666666666665,
      "calibration": 73.17734383695722,
      "cc": 23.802092840863022,
      "cc_by_type": {
       "score": 23.802092840863022
      },
      "cc_tiered": 19.097467638376042,
      "cc_tiered_open": 15.679388776063027,
      "cc_tiered_sealed": 22.51554650068906,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 23.802092840863022
     }
    },
    "raw_on_A": {
     "calibration": 67.24332350856244,
     "capability": 50.76538650167667,
     "intelligence": 34.2874494947909
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01866666666666672,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 17.125628095913363,
    "Emphasis on Accuracy 60:20:20": 13.343109026613647,
    "Emphasis on Cost 20:20:60": 19.23314758100122,
    "Emphasis on Speed 20:60:20": 20.730787178129347,
    "Intelligence only": 10.022590756029784
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "jaredpalmer",
   "axes": {
    "calibration": 86.77132815303852,
    "cost": null,
    "intelligence": 36.22478886598313,
    "speed": 89.01765799382792
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0745927868852459,
      "ece_score": 85.08144262295082,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 85.08144262295082,
      "score_before_time_limit": 85.08144262295082,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.420911528150134,
      "brier": 0.1571680953083108,
      "ece": 0.07976219839142094,
      "n": 373,
      "score": 84.0475603217158,
      "score_before_time_limit": 84.0475603217158,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9704301075268817,
      "interval90_width": 0.5497311827956989,
      "n": 372,
      "nrps": 0.10286757401209681,
      "score": 91.18498151444894,
      "score_before_time_limit": 91.18498151444894,
      "time_limit_failed_share": 0,
      "top_ece": 0.0367163978494623
     }
    },
    "score": 86.77132815303852
   },
   "capability": 61.49805850951083,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 3948e3f1..., and the later re-freezes, which only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base (the module RAISES 'base model lacks an explicit market-listing decision'); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens as reported by the author's server, kept as sent (input 662,682; output 164,737 is the author's billing-style count of the serialised answers, not generated text).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Kev-9B v2 (Jared Palmer, Qwen3.5-9B-Base LoRA + pointer head)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's kev.serve on torch.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 44.923827258334626,
    "I_sealed": 27.52575047363164,
    "base": 36.22478886598313,
    "excess": 7.558550014347276,
    "gap": 17.398076784702987,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 63.253539541495684,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 62.25228059138094,
       "judge": 68.35860250494397,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 24.727398205659078,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 46.79054402784911,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.67848484848486,
       "hard": 52.62685185185185,
       "judge": 44.33762711864405,
       "standard": 38.85333333333332
      }
     },
     "sealed|choice": {
      "cc": 55.791766173685986,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 71.41041931385007,
       "hard": 53.68335839019312,
       "judge": 52.39960822722822,
       "standard": 57.28749209027631
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 17.24137931034482,
       "hard": -14.54545454545455,
       "judge": -32.30769230769231,
       "standard": 19.402985074626855
      }
     },
     "sealed|score": {
      "cc": 26.785485247208936,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 51.87780701754385,
       "hard": 19.02600000000001,
       "judge": 22.344166666666666,
       "standard": 36.42027272727272
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "kev-9b-v2",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "listed",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "jaredpalmer/kev-9b@b5d8c18e44c60888d138b65cb6507ff0a5a448a0",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base and raises rather than estimating; one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.8888888888888888,
    "decisive": 216,
    "decisive_rate": 0.579088471849866,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.77132815303852,
     "cost": null,
     "intelligence": 36.22478886598313,
     "speed": 89.1575041920417
    },
    "board_release": "v1.6.43",
    "capability": 61.49805850951083,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25193482227623465,
     "p50_s_raw": 0.05096741113811731,
     "p95_s_adjusted": 0.481904943287372,
     "p95_s_raw": 0.165952471643686
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/jaredpalmer/kev-9b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2587409914471209,
    "p50_s_raw": 0.05437049572356045,
    "p95_s_adjusted": 0.484583879681304,
    "p95_s_raw": 0.16729193984065202
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 56.42555555555557,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 2.6100000000000234
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 56.42555555555557
      },
      "calibration_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 15.13716264581125,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": -12,
        "score": 17.856875000000016
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 15.13716264581125
      },
      "composite_gen": {
       "accuracy": 0.5445544554455446,
       "cc": 18.58461619530725,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -18.518518518518523,
        "score": 6.792692307692305
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 18.58461619530725
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 76.8387037037037,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 70.51611111111112
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 76.8387037037037
      },
      "jaggedness": {
       "accuracy": 0.39444444444444443,
       "cc": 1.3263832320119515,
       "cc_by_type": {
        "choice": 27.81520692974015,
        "noul": -51.35135135135135,
        "score": 27.51529411764705
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 1.3263832320119515
      },
      "llm_router": {
       "accuracy": 0.7348242811501597,
       "cc": 57.533374877889166,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 54.639175257731964,
        "score": 52.82931249999998
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 57.533374877889166
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5555555555555556,
       "cc": 6.022894779905092,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": -75,
        "score": 17.810952380952372
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 6.022894779905092
      },
      "multilingual": {
       "accuracy": 0.6036363636363636,
       "cc": 31.344664586196064,
       "cc_by_type": {
        "choice": 63.808281708509945,
        "noul": -13.888888888888884,
        "score": 44.11460093896714
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 31.344664586196064
      },
      "negation_pair_gen": {
       "accuracy": 0.37209302325581395,
       "cc": -16.322000000000006,
       "cc_by_type": {
        "choice": 36.66666666666667,
        "noul": -66.66666666666667,
        "score": -18.966000000000015
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 69.79409255409256,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 80,
        "score": 59.4123076923077
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 69.79409255409256
      },
      "usecase": {
       "accuracy": 0.5363636363636364,
       "cc": 27.856917647058825,
       "cc_by_type": {
        "choice": 55.88235294117649,
        "noul": 0,
        "score": 27.68839999999999
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 27.856917647058825
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 32.17107448107448,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": -15.391538461538467
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 32.17107448107448
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": -31.578947368421055,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 15.06477982590886,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": -0.7349999999999968
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 15.06477982590886
      },
      "de": {
       "accuracy": 0.875,
       "cc": 74.7749634474798,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 73.21869565217392
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 74.7749634474798
      },
      "el": {
       "accuracy": 0.4,
       "cc": 21.898075729360645,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.4
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.898075729360645
      },
      "en": {
       "accuracy": 0.577758470894874,
       "cc": 29.89460576976582,
       "cc_by_type": {
        "choice": 55.62660938968279,
        "noul": 4.316546762589923,
        "score": 29.74066115702476
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 29.89460576976582
      },
      "es": {
       "accuracy": 0.6774193548387096,
       "cc": 43.82127882800754,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 50,
        "score": -8.748235294117656
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 43.82127882800754
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 50.34222222222223,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 84.36000000000004
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 50.34222222222223
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 11.870773993808049,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": -100,
        "score": 77.96526315789474
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 11.870773993808049
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 19.22333333333332,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": -50,
        "score": 7.669999999999977
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 19.22333333333332
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 72.32631778198417,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 31.008571428571397
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 72.32631778198417
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 18.961827729238376,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": -19.999999999999996,
        "score": 19.776666666666674
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 18.961827729238376
      },
      "ko": {
       "accuracy": 0.2,
       "cc": -20.761772486772493,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": 26.603571428571414
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.20611111111111,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.61833333333334
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.20611111111111
      },
      "nl": {
       "accuracy": 0.3,
       "cc": -4.4380259740259715,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -100,
        "score": 64.608
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 32.91830097071453,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -19.999999999999996,
        "score": 41.60720930232558
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 32.91830097071453
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 43.445137339137375,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 19.999999999999996,
        "score": 52.76322580645163
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 43.445137339137375
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 16.34975177304965,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 45.645
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 16.34975177304965
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 29.102952898550726,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 100,
        "score": -47.47375
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 29.102952898550726
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 67.04629032258065,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 65.655
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.04629032258065
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 4.24764595103579,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -28.49999999999997
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 4.24764595103579
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6733333333333333,
       "cc": 57.806310874681884,
       "cc_by_type": {
        "choice": 57.806310874681884
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 57.806310874681884
      },
      "noul": {
       "accuracy": 0.512,
       "cc": 2.400000000000002,
       "cc_by_type": {
        "noul": 2.400000000000002
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 2.400000000000002
      },
      "score": {
       "accuracy": 0.496,
       "cc": 32.36452166064988,
       "cc_by_type": {
        "score": 32.36452166064988
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 32.36452166064988
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 0.5480392156862663,
       "cc_by_type": {
        "choice": 84,
        "noul": -50,
        "score": -32.3558823529412
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.5480392156862663
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 11.184166180035751,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -11.111111111111116,
        "score": 0.5642307692307891
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 11.184166180035751
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 11.229837398373975,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": -17.530000000000022
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 11.229837398373975
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -20.951396198830405,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": 7.3212500000000125
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 29.18038647342995,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 49.85999999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 29.18038647342995
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -112.2160272536688,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -100,
        "score": -273.7550000000001
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 33.790180277239095,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 14.28571428571428,
        "score": 26.30051282051282
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 33.790180277239095
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": -7.152472352673836,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100,
        "score": 30.80739130434782
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 22.21833333333333,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 28.321666666666655
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 22.21833333333333
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 28.726543269958686,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 33.33333333333333,
        "score": 29.934090909090916
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 28.726543269958686
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 38.53218855218855,
       "cc_by_type": {
        "choice": 100,
        "noul": 11.111111111111116,
        "score": 4.485454545454548
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 38.53218855218855
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -40.57075268817204,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": 7.319999999999993
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.4444444444444444,
       "cc": 17.760956849363755,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -19.999999999999996,
        "score": 30.864444444444427
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 17.760956849363755
      },
      "model_routing": {
       "accuracy": 0.7603686635944701,
       "cc": 62.05049783549784,
       "cc_by_type": {
        "choice": 70.7792207792208,
        "noul": 72.72727272727273,
        "score": 42.644999999999996
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 62.05049783549784
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 14.542993112792146,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 55.14529411764707
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 14.542993112792146
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 49.24341380377592,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 31.110000000000003
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 49.24341380377592
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 12.83834154351396,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -50,
        "score": 32.850000000000016
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 12.83834154351396
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": -1.5517004830917926,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -66.66666666666667,
        "score": 34.475333333333325
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 86.90343947056253,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100,
        "score": 82.06263157894738
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 86.90343947056253
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.15564648881318,
      87.27451843980873
     ],
     "I": [
      30.640212312460598,
      38.533368428432574
     ],
     "capability": [
      56.6085688585381,
      62.17252686389763
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 42
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.77132815303852,
     "capability": 61.49805850951083,
     "intelligence": 36.22478886598313
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6733333333333333,
      "calibration": 85.08144262295082,
      "cc": 57.806310874681884,
      "cc_by_type": {
       "choice": 57.806310874681884
      },
      "cc_tiered": 59.522652857590835,
      "cc_tiered_open": 63.253539541495684,
      "cc_tiered_sealed": 55.791766173685986,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.806310874681884
     },
     "noul": {
      "accuracy": 0.512,
      "calibration": 84.0475603217158,
      "cc": 2.400000000000002,
      "cc_by_type": {
       "noul": 2.400000000000002
      },
      "cc_tiered": 12.363699102829539,
      "cc_tiered_open": 24.727398205659078,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 2.400000000000002
     },
     "score": {
      "accuracy": 0.496,
      "calibration": 91.18498151444894,
      "cc": 32.36452166064988,
      "cc_by_type": {
       "score": 32.36452166064988
      },
      "cc_tiered": 36.788014637529024,
      "cc_tiered_open": 46.79054402784911,
      "cc_tiered_sealed": 26.785485247208936,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 32.36452166064988
     }
    },
    "raw_on_A": {
     "calibration": 85.59486213329072,
     "capability": 60.658649536353764,
     "intelligence": 35.722436939416816
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "knowledgator (Hugging Face)",
   "axes": {
    "calibration": 86.76027774140783,
    "cost": 70.90886953813762,
    "intelligence": 39.63748168273355,
    "speed": 85.63164752763427
   },
   "base_model": "Qwen/Qwen3.5-9B",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04536646565452951,
      "ece_score": 90.9267068690941,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 90.9267068690941,
      "score_before_time_limit": 90.9267068690941,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2975871313672922,
      "brier": 0.14111640148579332,
      "ece": 0.08224889439088115,
      "n": 373,
      "score": 83.55022112182378,
      "score_before_time_limit": 83.55022112182378,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9489247311827957,
      "interval90_width": 0.550179211469534,
      "n": 372,
      "nrps": 0.11097493704197242,
      "score": 85.80390523330561,
      "score_before_time_limit": 85.80390523330561,
      "time_limit_failed_share": 0,
      "top_ece": 0.08647347914595757
     }
    },
    "score": 86.76027774140783
   },
   "capability": 63.19887971207069,
   "code_revision": "The model card's own Jev-style decision API block ran unmodified (README serving block SHA256 bf3d0336b74da40b1f4f908eef7b9a6bed3a160a28d694320e5044160fa367c1). Native probabilities with no response shim or instruction conversion. gliclass 0.1.21, decoder-kv architecture, DecoderKVZeroShotClassificationPipeline; maximum joint input 32,768 tokens, no truncation, bf16 on one CUDA device, no quantisation. torch 2.14.0, transformers 5.20.0, tokenizers 0.23.3. Pure torch Qwen3.5 GatedDeltaNet path; no custom kernels. A first-party wrapper adds only a health route. Source and pinned safetensors weights ran on isolated rented hardware with read-only mounts, no credentials and networking disabled. The scored process was fresh.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 626,409 input + 0 output tokens over 1,485 decisions = 421.824242424 tokens/decision. GPU-time cost USD 0.00932619192740078/1,000; old list-price cost USD 0.0421824242424242/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00932619192740078/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.041760599999999995
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009326191927400784
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "First measurement of knowledgator/gliclass-v4-9b, a GLiClass model compatible with Jev-style decisions. Fine-tune of Qwen/Qwen3.5-9B: a decoder reads the question, a small recurrent DeBERTa-style scorer reads option labels, and a calibrator supplies native choice, noul and score probabilities. No generated explanation. scx-admin/Apex-large is a byte-identical republish of this checkpoint; the original repository was measured and the republish receives no separate row. The card's own serving configuration ran unchanged. Requests beyond the 32,768-token joint limit are refused before a forward pass. Of 1,500 requests, 1,485 answered and 15 returned HTTP 422 length refusals, scored as misses. No answered request was truncated. Native input usage equals an independent count on all answered requests; output is zero. No decision exceeded 10 s; loopback p95 was 0.25 s and maximum 0.48 s. The share view applies the common self-hosted latency assumption, twice measured p50 plus 0.15 s. A separate synthetic 32,768-token probe took 10.3–10.7 s on this hardware; the scored requests did not test performance near that length. Determinism record: 50/50 bit-identical repeat responses, zero argmax flips. The multimodal tower was loaded but no images were used. Card training-data statements and reported scores were not independently verified; training overlap with JevBench was not checked. Weights and base Apache-2.0; the original pinned repository has no separate LICENSE file, while the byte-identical republish includes the canonical Apache-2.0 text. One measured attempt; an earlier hardware create call failed before an instance existed.",
   "display": "GLiClass v4 9B (Knowledgator)",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-26",
    "release": "v1.7.9",
    "results_sha256": "0755266afdae3fb7c0098c96a783d998852e265dca6b329f0ebd1045969a0428"
   },
   "endpoint_condition": "Measured 26 Oct 2026 on one rented NVIDIA RTX A6000 48 GB in Finland, over loopback. One measured attempt with pinned weights and serving code in a fresh process, read-only model and source, no credentials and networking disabled. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 44.69832957213564,
    "I_sealed": 34.57663379333147,
    "base": 39.63748168273355,
    "excess": 0.28216900844846293,
    "gap": 10.121695778804174,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 68.71373175609203,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 60.36489462094998,
       "judge": 76.26895187870798,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 34.21187025534852,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 56.52173913043479,
       "judge": -9.999999999999998,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 31.16938670496637,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.5757834884421,
       "hard": 40.90112956656851,
       "judge": 37.20190055210585,
       "standard": -8.546068180684996
      }
     },
     "sealed|choice": {
      "cc": 58.74712250631529,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 69.50444726810673,
       "hard": 53.68335839019312,
       "judge": 55.92556317335946,
       "standard": 67.72832735709765
      }
     },
     "sealed|noul": {
      "cc": 17.404417507351123,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 37.93103448275863,
       "hard": 21.818181818181827,
       "judge": -4.61538461538461,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 27.578361366327997,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 61.07916296416972,
       "hard": 16.440315556750264,
       "judge": 26.432758701897296,
       "standard": 34.82245618320865
      }
     }
    }
   },
   "jevbench_score": 40.19829244906556,
   "key": "knowledgator_gliclass_v4_9b",
   "last_measured_on": "2026-10-26",
   "licence": "Apache-2.0 per the original model card (weights); the Apex republish includes the canonical Apache-2.0 LICENSE text; the original pinned repository has no separate LICENSE file; gliclass wheel Apache-2.0; base Qwen/Qwen3.5-9B Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.7.9",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "knowledgator/gliclass-v4-9b@14d11a92b5e0bab4a166e1fce4cc0777cac27182",
   "new_in": "v1.7.9",
   "noul_decisive": {
    "acc_among_decisive": 0.8893129770992366,
    "decisive": 262,
    "decisive_rate": 0.7024128686327078,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 146,
   "ranked": true,
   "ranks": {
    "A": 146,
    "B": 148,
    "C": 149,
    "capability": 142
   },
   "repo": "https://huggingface.co/knowledgator/gliclass-v4-9b",
   "scores": {
    "A": 40.19829244906556,
    "B": 35.8035836044877,
    "C": 27.915480867406643
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4194413520000012,
    "p50_s_raw": 0.1347206760000006,
    "p95_s_adjusted": 0.6518745149999632,
    "p95_s_raw": 0.2509372574999816
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 61.951627333958946,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 10.854882001876831
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 61.951627333958946
      },
      "calibration_gen": {
       "accuracy": 0.40594059405940597,
       "cc": 8.595546037534275,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": 12.00000000000001,
        "score": -6.679957857704721
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 8.595546037534275
      },
      "composite_gen": {
       "accuracy": 0.5247524752475248,
       "cc": 19.79112830336102,
       "cc_by_type": {
        "choice": 77.23577235772358,
        "noul": -3.703703703703709,
        "score": -14.158683743936805
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 19.79112830336102
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 80.42134304870733,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 81.26402914612198
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 80.42134304870733
      },
      "jaggedness": {
       "accuracy": 0.4722222222222222,
       "cc": 12.918414730327546,
       "cc_by_type": {
        "choice": 46.58325312800771,
        "noul": -35.13513513513513,
        "score": 27.307126198110055
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 12.918414730327546
      },
      "llm_router": {
       "accuracy": 0.7955271565495208,
       "cc": 68.31230152937509,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 77.31958762886597,
        "score": 58.52336609195277
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.31230152937509
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -11.375362118814024,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -50,
        "score": -9.899282232730734
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6581818181818182,
       "cc": 45.039967540373425,
       "cc_by_type": {
        "choice": 63.808281708509945,
        "noul": 30.555555555555557,
        "score": 40.75606535705477
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 45.039967540373425
      },
      "negation_pair_gen": {
       "accuracy": 0.5697674418604651,
       "cc": -1.1925129964947676,
       "cc_by_type": {
        "choice": 86.66666666666667,
        "noul": -46.666666666666664,
        "score": -43.57753898948431
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 76.42125789773054,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 100,
        "score": 65.29980972922765
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 76.42125789773054
      },
      "usecase": {
       "accuracy": 0.5772727272727273,
       "cc": 32.755442070682584,
       "cc_by_type": {
        "choice": 58.33333333333336,
        "noul": 4.166666666666674,
        "score": 35.76632621204773
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 32.755442070682584
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -4.09986535780888,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0,
        "score": -36.109119882950445
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 18.814980817936775,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 10.515602976083748
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 18.814980817936775
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 79.90199429219729,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 63.599788186326364
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 79.90199429219729
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 38.760620278983154,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 99.89452377922134
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 38.760620278983154
      },
      "en": {
       "accuracy": 0.6177237185056472,
       "cc": 36.461387296639174,
       "cc_by_type": {
        "choice": 60.898695402789784,
        "noul": 20.14388489208634,
        "score": 28.341581595041387
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 36.461387296639174
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 57.517245129333254,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": -7.872408168280831
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 57.517245129333254
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 50.432717323791785,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 83.53858912984529
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 50.432717323791785
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 32.78062433481964,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0,
        "score": 61.87128476916482
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 32.78062433481964
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 14.35622777607729,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": -23.411763599142432
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 14.35622777607729
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 71.36332561348955,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 28.11959492308751
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.36332561348955
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 45.218324848903514,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 80,
        "score": -1.4538419743378883
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 45.218324848903514
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 63.09672713028107,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100,
        "score": 33.734625835287645
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 63.09672713028107
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 72.1628165507546,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 99.82178298559712
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 72.1628165507546
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 16.68067565023856,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 35.32341223210096
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 16.68067565023856
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 59.18301085246353,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 52.78390348417857
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 59.18301085246353
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 64.63580134641553,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": 46.94217127554595
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 64.63580134641553
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 38.621592007524576,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 19.999999999999996,
        "score": 46.92860580980778
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 38.621592007524576
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": 20.869446256994337,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 6.086599640548229
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 20.869446256994337
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 61.35969291371043,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 16.337143257260323
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 61.35969291371043
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -9.77881933613321,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -45.155667047947645
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7026666666666667,
       "cc": 61.595131938996154,
       "cc_by_type": {
        "choice": 61.595131938996154
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 61.595131938996154
      },
      "noul": {
       "accuracy": 0.6213333333333333,
       "cc": 24.26666666666666,
       "cc_by_type": {
        "noul": 24.26666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 24.26666666666666
      },
      "score": {
       "accuracy": 0.49066666666666664,
       "cc": 30.401496183786712,
       "cc_by_type": {
        "score": 30.401496183786712
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 30.401496183786712
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 14.342024513611603,
       "cc_by_type": {
        "choice": 52,
        "noul": 0,
        "score": -8.973926459165193
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 14.342024513611603
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 15.640055178359331,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 33.33333333333333,
        "score": -11.879006307572105
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 15.640055178359331
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 48.472431239477665,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50,
        "score": 60.45794412493706
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 48.472431239477665
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -10.883649955657958,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -33.333333333333336,
        "score": -40.83808436989784
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 21.65437990997509,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 27.2819803096354
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 21.65437990997509
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -12.686145985869992,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -33.333333333333336,
        "score": -16.674790158867836
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 21.969653191735905,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -42.85714285714286,
        "score": 38.177867138232926
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 21.969653191735905
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": 0.08133884462150813,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -50,
        "score": 33.86770991365546
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.08133884462150813
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 23.885742232755376,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 33.32389336493281
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 23.885742232755376
      },
      "knowledge_graphs": {
       "accuracy": 0.45454545454545453,
       "cc": 10.312527411827462,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 0,
        "score": 8.02537666803056
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 10.312527411827462
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 38.01677086591353,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 36.7893306855959
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 38.01677086591353
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -19.284385835291236,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": 6.662971526384354
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 46.05012513803491,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 29.666690193375356
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 46.05012513803491
      },
      "model_routing": {
       "accuracy": 0.8387096774193549,
       "cc": 70.41293046306309,
       "cc_by_type": {
        "choice": 83.63636363636363,
        "noul": 78.78787878787878,
        "score": 48.814548964946866
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 70.41293046306309
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 41.57229379612094,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 0,
        "score": 70.78214050544535
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 41.57229379612094
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 50.40376064709008,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 51.30413245194807
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.40376064709008
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 20.367475925053903,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50,
        "score": 25.88075289831442
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 20.367475925053903
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 34.78364862515259,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 0,
        "score": 33.33645312183457
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 34.78364862515259
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 92.768368342891,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 88.9812616123029
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 92.768368342891
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.03378427438507,
      87.617083263968
     ],
     "I": [
      33.190263850139424,
      44.16871376880447
     ],
     "capability": [
      58.081192293735825,
      65.26288477621355
     ],
     "composite": null,
     "seed": 43
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.76027774140783,
     "capability": 63.19887971207069,
     "intelligence": 39.63748168273355
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7026666666666667,
      "calibration": 90.9267068690941,
      "cc": 61.595131938996154,
      "cc_by_type": {
       "choice": 61.595131938996154
      },
      "cc_tiered": 63.73042713120366,
      "cc_tiered_open": 68.71373175609203,
      "cc_tiered_sealed": 58.74712250631529,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.595131938996154
     },
     "noul": {
      "accuracy": 0.6213333333333333,
      "calibration": 83.55022112182378,
      "cc": 24.26666666666666,
      "cc_by_type": {
       "noul": 24.26666666666666
      },
      "cc_tiered": 25.80814388134982,
      "cc_tiered_open": 34.21187025534852,
      "cc_tiered_sealed": 17.404417507351123,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.26666666666666
     },
     "score": {
      "accuracy": 0.49066666666666664,
      "calibration": 85.80390523330561,
      "cc": 30.401496183786712,
      "cc_by_type": {
       "score": 30.401496183786712
      },
      "cc_tiered": 29.373874035647184,
      "cc_tiered_open": 31.16938670496637,
      "cc_tiered_sealed": 27.578361366327997,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.401496183786712
     }
    },
    "raw_on_A": {
     "calibration": 83.12890952708962,
     "capability": 61.11476015710933,
     "intelligence": 39.10061078712904
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 36.96111078282012,
    "Emphasis on Accuracy 60:20:20": 30.968455324988273,
    "Emphasis on Cost 20:20:60": 39.667788498319595,
    "Emphasis on Speed 20:60:20": 42.25453651251611,
    "Intelligence only": 24.91025391151059
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "PelaAI (Hugging Face)",
   "axes": {
    "calibration": 89.44042524156703,
    "cost": 62.33805091041278,
    "intelligence": 65.23825939945094,
    "speed": 89.461278844294
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.060334629789288274,
      "ece_score": 87.93307404214235,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 86.76063305491378,
      "score_before_time_limit": 87.93307404214235,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.19999999999999996,
      "brier": 0.0781421543165471,
      "ece": 0.049967061089730265,
      "n": 375,
      "score": 89.52655264721632,
      "score_before_time_limit": 90.00658778205394,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.96,
      "interval90_width": 0.26199999999999996,
      "n": 375,
      "nrps": 0.05710280076252524,
      "score": 92.034090022571,
      "score_before_time_limit": 92.77630042597882,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.04368559535894919
     }
    },
    "score": 89.44042524156703
   },
   "capability": 77.33934232050899,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,531,517 input + 1,485 output tokens over 1,485 decisions = 1032.32457912 tokens/decision. GPU-time cost USD 0.0180052146411404/1,000; old list-price cost USD 0.0931192121212121/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0180052146411404/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.16365029999999997
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.018005214641140434
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of this serving version. SAME WEIGHTS AS Gen2 (no earlier version of this exact model is on the board): the shards, tokenizer, configuration and chat template are byte-identical to the predecessor; the new author layout reads short text inputs twice, so this measures a new serving version, not new weights. AUTHOR'S OWN SERVER on our rented hardware, bf16. NATIVE OUTPUTS: label-token logprobs, label-only softmax at temperature 1; choice argmax, Noul P(true), score expectation; header off, no rotation or calibration file. AUTHOR EXPOSURE (card, self-reported, not verified): the authors adopted read twice after checking their own evaluation sets and a fresh full run of the Decision Index 0.3 public suite; the card also reports JevBench / JevBench-hard evaluations. Training-data, Decision Index format-data and decontamination disclosures are inherited from Gen2; no overlap or contamination audit was run by us. The authors say no Jev or other decision-model outputs were used as training labels. The author's H20 timing does not transfer to this RTX PRO 6000 measurement. 10 s RULE: 15 decisions took 12.0-13.7 s (maximum 13.691 s), exactly the 15 standard very long positions, and are scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit) (median 0.0789 s, p95 0.1089 s). DETERMINISM: first 50 positions run twice, 0/50 bit-identical, 0 argmax flips, maximum probability difference 0.0077745; bit reproducibility is not established and the cause was not investigated. CONTEXT: SGLang configuration limit 262,144 tokens; no author-imposed lower cap, no truncation and 0 refusals; the longest item was 81,706 tokens. The measurement runner's internal time cap fired about 1.4 seconds before the scored loop ended. An independent review found the answer file complete: all 1,500 requests were answered, with token usage equal to the offline count. The runner's partial flag is retained in the measurement record. The answer file was lost on 26 Oct 2026 after scoring. These are the scores and confidence intervals computed on 23 Oct; the measurement cannot be scored again. Subject-topic and use-case breakdowns are unavailable. The Noul decisiveness diagnostic is unavailable because the stored summaries do not record which timed-out answers were decisive.",
   "display": "KnowLine 26B-A4B-Gen2.5",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.18",
    "results_sha256": "900b71c83b730781d486e89da35e5aaa12ad38ef8a1d534366fb9b7a755e5dba",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Self-hosted on one evaluator-rented NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB GPU in Finland (EU), using the author's own server over SGLang 0.5.21. Contributor code ran in an offline container on a disposable rented GPU VM. Measured 23 Oct 2026. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 67.91192188060225,
    "I_sealed": 62.564596918299635,
    "base": 65.23825939945094,
    "excess": -4.492201808053096,
    "gap": 5.347324962302615,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.20538674872894,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 67.9144385026738,
       "judge": 80.22412656558998,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 64.26363008971704,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 73.91304347826086,
       "judge": 39.99999999999999,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 64.2667488033608,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 57.549934597763944,
       "hard": 69.74300327096145,
       "judge": 62.79683330161001,
       "standard": 58.87752022358408
      }
     },
     "sealed|choice": {
      "cc": 79.14521857388178,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 75.02023823291314,
       "judge": 77.08129285014691,
       "standard": 81.01666315123391
      }
     },
     "sealed|noul": {
      "cc": 49.62008860928058,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 38.18181818181819,
       "judge": 53.846153846153854,
       "standard": 58.20895522388059
      }
     },
     "sealed|score": {
      "cc": 58.928483571736564,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 83.24526303104611,
       "hard": 52.735969930554404,
       "judge": 59.11975655197289,
       "standard": 58.86821165409162
      }
     }
    }
   },
   "jevbench_score": 74.44818685146753,
   "key": "knowline-26b-a4b-gen2.5",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 weights (model card and LICENSE); MIT author serving code (file headers); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.18",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": {
    "base": "google/gemma-4-26B-A4B-it",
    "model": "PelaAI/KnowLine-26B-A4B-Gen2.5",
    "revision": "7486767f8546beb510e61aef07d0e74219ceb4e3"
   },
   "new_in": "v1.7.18",
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive": null,
    "decisive_rate": null,
    "n": 375,
    "supported": true,
    "unavailable_reason": "The Noul decisiveness diagnostic is unavailable because the stored summaries do not record which timed-out answers were decisive.",
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "provider": "self-hosted",
   "rank": 26,
   "ranked": true,
   "ranks": {
    "A": 26,
    "B": 33,
    "C": 23,
    "capability": 49
   },
   "repo": "https://huggingface.co/PelaAI/KnowLine-26B-A4B-Gen2.5",
   "scores": {
    "A": 74.44818685146753,
    "B": 72.4038818523965,
    "C": 74.44818685146753
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.30785713999980546,
    "p50_s_raw": 0.07892856999990272,
    "p95_s_adjusted": 0.36772477010008514,
    "p95_s_raw": 0.10886238505004257
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 82.42681835400788,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 47.28045506202365
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 82.42681835400788
      },
      "calibration_gen": {
       "accuracy": 0.801980198019802,
       "cc": 66.04758081543544,
       "cc_by_type": {
        "choice": 77.7306468716861,
        "noul": 52,
        "score": 68.4120955746202
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 66.04758081543544
      },
      "composite_gen": {
       "accuracy": 0.8811881188118812,
       "cc": 82.46190984237806,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 77.77777777777777,
        "score": 72.85998426968159
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 82.46190984237806
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.74772374855797,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 97.24317124567393
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.74772374855797
      },
      "jaggedness": {
       "accuracy": 0.6777777777777778,
       "cc": 48.252828535303024,
       "cc_by_type": {
        "choice": 63.90760346487007,
        "noul": 24.32432432432432,
        "score": 56.52655781671469
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 48.252828535303024
      },
      "llm_router": {
       "accuracy": 0.8434504792332268,
       "cc": 72.82171159544708,
       "cc_by_type": {
        "choice": 81.77335563969358,
        "noul": 77.31958762886597,
        "score": 59.37219151778168
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 72.82171159544708
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 37.10195929494652,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 0,
        "score": 48.41927994669521
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 37.10195929494652
      },
      "multilingual": {
       "accuracy": 0.7563636363636363,
       "cc": 57.20874085126767,
       "cc_by_type": {
        "choice": 76.5242908379524,
        "noul": 33.33333333333333,
        "score": 61.76859838251729
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 57.20874085126767
      },
      "negation_pair_gen": {
       "accuracy": 0.9069767441860465,
       "cc": 76.1179112736941,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 68.35373382108229
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 76.1179112736941
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 83.67312376651775,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 66.0343863145683
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 83.67312376651775
      },
      "usecase": {
       "accuracy": 0.7363636363636363,
       "cc": 58.516111821978996,
       "cc_by_type": {
        "choice": 75.49019607843138,
        "noul": 45.83333333333333,
        "score": 54.22480605417229
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 58.516111821978996
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 49.17412126456687,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50,
        "score": 16.569982841319675
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 49.17412126456687
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 34.896670211176506,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": 12.831331678076396
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 34.896670211176506
      },
      "de": {
       "accuracy": 0.875,
       "cc": 78.58827824000367,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 84.65864002974554
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 78.58827824000367
      },
      "el": {
       "accuracy": 0.6,
       "cc": 49.81358741844482,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 0,
        "score": 99.720091864273
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 49.81358741844482
      },
      "en": {
       "accuracy": 0.8045178105994787,
       "cc": 66.80651461053104,
       "cc_by_type": {
        "choice": 80.01000720030265,
        "noul": 58.99280575539569,
        "score": 61.41673087589478
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 66.80651461053104
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 74.53246502234954,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 33.38532328890835
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 74.53246502234954
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 76.28528400341526,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 95.52251867691245
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 76.28528400341526
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 85.78648096931327,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 78.53591349617508
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 85.78648096931327
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 56.7659401008144,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 50,
        "score": 37.05759683875603
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 56.7659401008144
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 79.47247577825993,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 38.417427334779816
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 79.47247577825993
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 18.55828836239586,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 0,
        "score": -1.4339514338608694
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 18.55828836239586
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 29.406192643578038,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 88.21857793073413
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 29.406192643578038
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.25644796184417,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.76934388553252
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.25644796184417
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 9.278211159982488,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": -12.858007212693234
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.278211159982488
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 64.65541780870437,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 69.20112435290108
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 64.65541780870437
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 48.00173883664217,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 19.999999999999996,
        "score": 66.43303029896599
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 48.00173883664217
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 34.6563643958734,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 19.999999999999996,
        "score": 35.03292297485425
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 34.6563643958734
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 66.36889202463412,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 20.845806508684962
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 66.36889202463412
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 85.61244071803405,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 89.09538667023116
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 85.61244071803405
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 52.61153173951366,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 24.501261885207658
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 52.61153173951366
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8386666666666667,
       "cc": 79.16148414627146,
       "cc_by_type": {
        "choice": 79.16148414627146
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 79.16148414627146
      },
      "noul": {
       "accuracy": 0.7706666666666667,
       "cc": 54.13333333333335,
       "cc_by_type": {
        "noul": 54.13333333333335
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 54.13333333333335
      },
      "score": {
       "accuracy": 0.728,
       "cc": 60.685313662984505,
       "cc_by_type": {
        "score": 60.685313662984505
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 60.685313662984505
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -2.258749844064669,
       "cc_by_type": {
        "choice": 36,
        "noul": -50,
        "score": 7.223750467805989
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 77.26723755904693,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 50.43525304981159
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 77.26723755904693
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 17.99220279450602,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": 2.757096188396113
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 17.99220279450602
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -1.7498041837774287,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -33.333333333333336,
        "score": -13.436547054256254
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 29.736293007329532,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 51.52771960169873
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 29.736293007329532
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 45.302236521008474,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 66.66666666666667,
        "score": 6.975891952962532
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 45.302236521008474
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 31.5590980810032,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": -42.85714285714286,
        "score": 47.338358668779904
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 31.5590980810032
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 44.36953928804933,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0,
        "score": 64.46750288156959
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 44.36953928804933
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 56.114015978286375,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 70.84204793485911
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 56.114015978286375
      },
      "knowledge_graphs": {
       "accuracy": 0.7727272727272727,
       "cc": 66.20965161603672,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 44.81025392020154
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 66.20965161603672
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 50.20537870390492,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 33.33333333333333,
        "score": 40.538616731869816
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 50.20537870390492
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 31.141440948280856,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 24.607118543767303
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 31.141440948280856
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 59.97011606735634,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 71.42666298133963
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.97011606735634
      },
      "model_routing": {
       "accuracy": 0.8663594470046083,
       "cc": 74.35156021785807,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 84.84848484848484,
        "score": 47.55684515573869
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.35156021785807
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 19.009553429434163,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 68.54497506757312
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 19.009553429434163
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 60.23258084035714,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 64.07750110974364
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 60.23258084035714
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 29.80830736745914,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 50,
        "score": -16.240102528164435
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 29.80830736745914
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 36.73139878344289,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 53.6724572198939
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 36.73139878344289
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.77870020143457,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.33610060430377
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.77870020143457
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.73395590486754,
      90.43908478179783
     ],
     "I": [
      60.150376207494695,
      69.26419614481709
     ],
     "capability": [
      73.08864852835562,
      79.06874532802999
     ],
     "composite": null,
     "seed": 43
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.44042524156703,
     "capability": 77.33934232050899,
     "intelligence": 65.23825939945094
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8386666666666667,
      "calibration": 86.76063305491378,
      "cc": 79.16148414627146,
      "cc_by_type": {
       "choice": 79.16148414627146
      },
      "cc_tiered": 77.17530266130535,
      "cc_tiered_open": 75.20538674872894,
      "cc_tiered_sealed": 79.14521857388178,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 79.16148414627146
     },
     "noul": {
      "accuracy": 0.7706666666666667,
      "calibration": 89.52655264721632,
      "cc": 54.13333333333335,
      "cc_by_type": {
       "noul": 54.13333333333335
      },
      "cc_tiered": 56.94185934949881,
      "cc_tiered_open": 64.26363008971704,
      "cc_tiered_sealed": 49.62008860928058,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 54.13333333333335
     },
     "score": {
      "accuracy": 0.728,
      "calibration": 92.034090022571,
      "cc": 60.685313662984505,
      "cc_by_type": {
       "score": 60.685313662984505
      },
      "cc_tiered": 61.59761618754868,
      "cc_tiered_open": 64.2667488033608,
      "cc_tiered_sealed": 58.928483571736564,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 60.685313662984505
     }
    },
    "raw_on_A": {
     "calibration": 85.6109347807397,
     "capability": 75.39024718412394,
     "intelligence": 65.16955958750818
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.50857610442917,
    "Emphasis on Accuracy 60:20:20": 68.30146419770846,
    "Emphasis on Cost 20:20:60": 66.99615464477206,
    "Emphasis on Speed 20:60:20": 77.0367874193333,
    "Intelligence only": 65.23825939945094
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "PelaAI (Hugging Face)",
   "axes": {
    "calibration": 92.98946060661117,
    "cost": 46.490946925756504,
    "intelligence": 77.43747194470296,
    "speed": 81.4421691177115
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.022168923936636005,
      "ece_score": 95.5662152126728,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 94.29199900983717,
      "score_before_time_limit": 95.5662152126728,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.09599999999999997,
      "brier": 0.047840463085377406,
      "ece": 0.02830715981688726,
      "n": 375,
      "score": 93.8354290070939,
      "score_before_time_limit": 94.33856803662255,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9466666666666667,
      "interval90_width": 0.12422222222222223,
      "n": 375,
      "nrps": 0.042848436403401725,
      "score": 90.8409538029025,
      "score_before_time_limit": 91.57354214002268,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.06284036039807225
     }
    },
    "score": 92.98946060661117
   },
   "capability": 85.21346627565707,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10271.2448706 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=native). The row's own scored cost basis: 1,531,517 input + 1,485 output tokens over 1,485 decisions = 1032.32457912 tokens/decision. GPU-time cost USD 0.0607624256223566/1,000; old list-price cost USD 0.0931592121212121/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0607624256223566/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.16369029999999998
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.06076242562235659
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "KnowLine 31B-Gen1",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5991917808219176/h recorded by the runner. One VM for this row only (attempt 2; attempt 1 used a separate VM). SGLang (127.0.0.1:9080) and author front end (127.0.0.1:8000) in ONE --network none --read-only container; driver requests serial over loopback. Read-only code, model and input, blank credential variables, HF offline flags. In-container network cut proven before sealed input: only loopback, no default route, DNS and TCP fail; code inventory/hashes and model files checked; weights re-hashed against the manifest (24 files, 0 failures). SGLang /server_info sizing gate passed before sealed input (SERVER_SIZING_OK: version 0.5.21, swa_full_tokens_ratio 0.15, mem_fraction_static 0.85, dtype auto, kv_cache_dtype auto, no quantization, FP32 LM head on, max_total_num_tokens 106,983, max_req_input_len 106,977). Attempt 2 measured 24 Oct 2026 (reservation 09:49:54Z, VM created 09:51:04Z, teardown confirmed 10:25:27Z); this row's measurement container 1,401 s, run cap 5,036 s, complete=true, deadline_hit=false, rc 0; scored loop 1,252.397 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 54da5a6239bb6d60a77d5fbed9541c4ed3aded49f9e6f1961d46e1826cc9b323. Node-shred receipt verified; row cleanup verified; exact-id VM and volume teardown confirms provider absent and guard closed. GPU rental is not the benchmark's estimated model tariff. ",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 78.65070323008781,
    "I_sealed": 76.2242406593181,
    "base": 77.43747194470296,
    "excess": -7.413064199585996,
    "gap": 2.426462570769715,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 81.5623739827074,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 79.2387543252595,
       "judge": 76.26895187870798,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 79.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 70,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 74.79070189113094,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 79.802915192265,
       "hard": 83.44397984101029,
       "judge": 78.30304801907488,
       "standard": 49.709520148889276
      }
     },
     "sealed|choice": {
      "cc": 85.87812548753575,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 83.34682548860874,
       "judge": 85.89618021547503,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 72.25099964369136,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 60.00000000000001,
       "judge": 75.38461538461539,
       "standard": 85.07462686567165
      }
     },
     "sealed|score": {
      "cc": 70.54359684672721,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 89.28102570375478,
       "hard": 72.52297440840516,
       "judge": 65.03461934934525,
       "standard": 65.4795935409305
      }
     }
    }
   },
   "jevbench_score": 60.18983325088006,
   "key": "knowline-31b-gen1",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 weights (model card and LICENSE); MIT author serving code (file headers); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "PelaAI/KnowLine-31B-Gen1@986803e4037f7778d24623184729a31202e3d302",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9733727810650887,
    "decisive": 338,
    "decisive_rate": 0.9061662198391421,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 109,
   "ranked": true,
   "ranks": {
    "A": 109,
    "B": 104,
    "C": 85,
    "capability": 8
   },
   "repo": "https://huggingface.co/PelaAI/KnowLine-31B-Gen1",
   "scores": {
    "A": 60.18983325088006,
    "B": 61.430331565700484,
    "C": 60.18983325088006
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.6593511499998158,
    "p50_s_raw": 0.2546755749999079,
    "p95_s_adjusted": 1.088093761300297,
    "p95_s_raw": 0.4690468806501485
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run unmodified: PelaAI/KnowLine-31B-Gen1 at 986803e4037f7778d24623184729a31202e3d302, knowline_server.py sha256 13ca50c4001fac0aa3899a35ff01e60f311132c571b17a8c3dc7aa6001ad2f3f (1,156 lines) and knowline_layout.py sha256 2e839fa24f1ad9c980898b2a9aa62e2154ba3f57618bd5f2fedb5756f61f062f (242 lines), plus the data file temperatures.json sha256 463a46c5ea253649bd5f43809655637d2d41939e503c81ddd01b28355d736f62 ({\"score\": 0.6944}), all mounted read-only at /model over stock SGLang 0.5.21. Only these two author Python files shipped; knowline_engine.py and serve_knowline.sh were excluded and never executed (the SGLang flags follow the author's serve_knowline.sh, plus our one pooling flag --swa-full-tokens-ratio 0.15, see below). Commands: env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONDONTWRITEBYTECODE=1 python3 -m sglang.launch_server --model-path /model --served-model-name m --tp 1 --mem-fraction-static 0.85 --enable-fp32-lm-head --swa-full-tokens-ratio 0.15 --host 127.0.0.1 --port 9080 ; env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONDONTWRITEBYTECODE=1 python3 /model/knowline_server.py --model /model --backend sglang --url http://127.0.0.1:9080 --served-model-name knowline-31b-gen1 --temperature 1 --workers 16 --read-twice --twice-max-tokens 3000 --host 127.0.0.1 --port 8000 --temperatures /model/temperatures.json --box-prompt gemma --box-coords yxyx. Never passed: --allow-url-assets, --referenced-fields, --header, --no-read-twice, --debug-refusals, --box-calibration, --box-prompt-template (asserted on the rendered launch script; /health after the measurement shows referenced_fields false, url_assets false, no box calibration). Image: lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469, derived image sha256:579dd709d9118e745c9be62fc134fc6aac272a2ee75834da15f7cd235a025a4e (only blank-credential ENV, no package installs). Python 3.12.3, sglang 0.5.21, torch 2.13.0+cu130, transformers 5.12.1, tokenizers 0.22.2, flashinfer-python 0.6.18, sglang-kernel 0.4.7. Code gate: cleared by the benchmark owner on the disclosed patterns. DISCLOSED PATTERNS: a new dormant runtime download path, requests.get of http(s) image URLs (knowline_server.py _fetch, line 383), reachable only behind the command-line flag --allow-url-assets (off by default, never passed, container without network); dormant Hub-capable from_pretrained loaders in the unused --backend hf class; served tokenizer load from local /model under offline flags; HTTP client exclusively to loopback SGLang /generate (scoring and the dormant box path); local listening socket on 127.0.0.1. About 800 added lines (image, box and referenced-fields code) are not reachable from text-only requests; the one text-path change (noul criteria values rendered through render_value) leaves string, null and number criteria byte-identical and is mirrored by our offline token counter. Delta source review PASS WITH CONDITIONS (flag discipline, hash-checked snapshot, in-container offline proof, all enforced by the runner), the runner was reviewed and findings fixed. OUR ONE LAUNCH-FLAG CHANGE (pre-registration Amendment 3): attempt 1 stopped in the synthetic smoke because the default SGLang pool sizing (--swa-full-tokens-ratio 0.8) gave only 29,717 full-attention cache tokens; attempt 2 adds --swa-full-tokens-ratio 0.15, which only reallocates the same KV memory between the full-attention and sliding-window pools (live capacity max_total_num_tokens 106,983, max_req_input_len 106,977); no change to weights, BF16 precision, KV dtype, mem-fraction, context configuration or the 1,024-token attention window; bit identity against the default sizing is not promised. The amendment was reviewed and the final hashes checked (final runner sha256 18c051f1b72ec682339a11eb90025eb4c73fc4b526932c24f3faecf557fc270d), covered by the standing clearance for this amendment. Weights: Apache-2.0, public, ungated, 62,546,336,352 weight bytes in 13 BF16 shards, hash-checked file by file on the node; prereg manifest sha256 0444b046991995634646ef018d16103292eb9807147d0287898b2380b95ed98d; the shards share no LFS sha256 with the weights of the measured KnowLine 4B, 26B and 35B rows. FIRST MEASUREMENT of KnowLine-31B-Gen1 (our own intake). FIRST DENSE-GEMMA KnowLine: new weights (Gemma 4 31B-it fine-tune, LoRA rank 32 merged, then averaged 0.75 fine-tuned + 0.25 base per tensor, readout temperature 1.44 folded into the final norm and soft-cap), not a new serving version of an earlier row; the 4B, 26B and 35B KnowLine rows are different weights. AUTHOR'S OWN SERVER on our rented hardware, bf16 (no FP8; the author documents bf16 only). The server and layout files are not the bytes measured for the earlier KnowLine rows (1,156 lines against 367; see code_revision); on our text items the prompts follow the same read-twice layout. NATIVE OUTPUTS: label-token logprobs, label-only softmax at temperature 1 for choice and noul questions and at the author's 0.6944 for score questions (temperatures.json, effective 1.0 on top of the folded 1.44; both are ranking-relevant behaviours of the author's server); choice argmax, Noul P(true), score expectation; header off, no rotation, no calibration file. AUTHOR EXPOSURE (card, self-reported, not verified): Decision Index 0.3 public suite 70.13, self-run read twice; the card also reports JevBench / JevBench-hard evaluations; the training data is that of KnowLine-26B-A4B-Gen1 (906,122 rows, about 33% targeting Decision Index benchmarks in their request format, including train splits of public datasets rewritten in that format and synthetic items in the same format); the authors checked it against the Decision Index 0.3 rows, their own evaluation sets and public benchmarks including JevBench and removed 491 near-duplicate rows; the 0.75 averaging weight was chosen on the authors' own evaluation sets; LLM teachers labelled the synthetic tasks and not all labels were checked by a human; the card says part of the public-suite score comes from benchmarks whose tasks are in the training data. The authors say no Jev or other decision-model outputs were used as training labels. No overlap or contamination audit was run by us. The author's H20 timing (340 ms median, 3,977 ms at the 95th percentile) does not transfer to this RTX PRO 6000 measurement. CONTEXT: SGLang configuration limit 262,144 tokens, with live KV capacity max_total_num_tokens 106,983 and max_req_input_len 106,977 at our pooling flag; no author-imposed lower cap, no truncation and 0 refusals; the longest item was 81,706 tokens. 10 s RULE: exactly 15 decisions over 10 s, the standard very long items at positions in the pre-registered long-item set, cold prefills of 76,531-81,706 tokens, up to 59.753 s each; these count wrong in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit). No decision came near the adapter's 120 s timeout. Maximum 59.753078 s, median 0.255530 s, p95 0.469112 s. DETERMINISM: first 50 positions run twice, 40/50 bit-identical, 0 argmax flips, maximum probability difference 0.0141815; the scored raw is the first pass, so scores are unaffected; bit reproducibility is not established and the cause was not investigated. ATTEMPT HISTORY: attempt 1, pod window 24 Oct 2026 08:55-09:07Z (about USD 0.49 on the runner ledger) on a separate disposable GPU instance, stopped in the synthetic smoke before any held-out item: at the default pool sizing SGLang reported 29,717 full-attention cache tokens and refused the 32,000-token probe (HTTP 422, maximum allowed input length 29,711). No sealed byte was sent and exposure stayed unspent; the attempt is retained as a receipt only, never scored or merged. pre-registration Amendment 3 changed only our SGLang launch flag (--swa-full-tokens-ratio 0.15) and added a server_info sizing gate (max_total_num_tokens >= 90,000, max_req_input_len >= 85,000) that precedes the first sealed byte; a further review and a final-hash check preceded the retry. evaluator exposure and device ruling covered attempt 1 and ruling on file admitted attempt 2 as the final attempt; evaluator exposure and device ruling was unspent at attempt 1 and is consumed by attempt 2. Attempt 2 is this measurement on new a separate disposable GPU instance, 24 Oct 09:49-10:25Z, about USD 1.54 rental. Between attempts only the pooling flag and our first-party runner and tests changed; the scoring harness, model, author code, input and all other engine settings are unchanged, and the pinned image stack was rebuilt on the new VM. Over the cost cap under the earlier list-price basis; inside both caps from v1.8.0 (GPU-time cost basis).",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 88.3968677123802,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 65.1906031371406
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 88.3968677123802
      },
      "calibration_gen": {
       "accuracy": 0.8217821782178217,
       "cc": 71.16448338121529,
       "cc_by_type": {
        "choice": 84.09331919406152,
        "noul": 52,
        "score": 77.40013094958434
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 71.16448338121529
      },
      "composite_gen": {
       "accuracy": 0.9405940594059405,
       "cc": 90.76705927078392,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 82.9606177400844
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 90.76705927078392
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.99941727196868,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.99825181590603
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.99941727196868
      },
      "jaggedness": {
       "accuracy": 0.8277777777777777,
       "cc": 71.83292503617992,
       "cc_by_type": {
        "choice": 76.90086621751685,
        "noul": 56.75675675675676,
        "score": 81.84115213426612
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 71.83292503617992
      },
      "llm_router": {
       "accuracy": 0.9424920127795527,
       "cc": 83.94560601208627,
       "cc_by_type": {
        "choice": 96.83014880690322,
        "noul": 93.81443298969072,
        "score": 61.19223623966488
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.94560601208627
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 50.456047188010906,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 50,
        "score": 38.48154362588839
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 50.456047188010906
      },
      "multilingual": {
       "accuracy": 0.8254545454545454,
       "cc": 70.76021074907139,
       "cc_by_type": {
        "choice": 81.41506358004565,
        "noul": 58.33333333333333,
        "score": 72.5322353338352
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 70.76021074907139
      },
      "negation_pair_gen": {
       "accuracy": 0.9302325581395349,
       "cc": 85.90181803064407,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 86.66666666666667,
        "score": 77.7054540919322
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 85.90181803064407
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.9375,
       "cc": 92.74806434340171,
       "cc_by_type": {
        "choice": 90.99099099099098,
        "noul": 100,
        "score": 87.25320203921414
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 92.74806434340171
      },
      "usecase": {
       "accuracy": 0.7954545454545454,
       "cc": 72.0418358087616,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 70.83333333333333,
        "score": 71.02746821059853
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 72.0418358087616
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 53.48166703504766,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100,
        "score": 17.58785824800013
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 53.48166703504766
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 71.22869587423908,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 34.42341481165732
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 71.22869587423908
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 88.06098042697631,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 88.07674659066348
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 88.06098042697631
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.60456618679116,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.99991829966588
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.60456618679116
      },
      "en": {
       "accuracy": 0.8809730668983493,
       "cc": 80.4997741482516,
       "cc_by_type": {
        "choice": 86.60011471668639,
        "noul": 79.85611510791367,
        "score": 75.04309262015474
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 80.4997741482516
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 72.48344614313717,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 27.238266651271214
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 72.48344614313717
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 89.07078777126578,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": 99.99924855969905
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 89.07078777126578
      },
      "fr": {
       "accuracy": 1,
       "cc": 98.0514903309848,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 94.15447099295446
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 98.0514903309848
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 80.47847958302002,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 75,
        "score": 66.43543874906005
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 80.47847958302002
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 63.17653605018162,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 3.5592262331637037
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 63.17653605018162
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 46.005914007082026,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 39.99999999999999,
        "score": 12.314803180896583
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 46.005914007082026
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 31.04667785453655,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 93.14003356360965
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 31.04667785453655
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.77688190192204,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.99731237243283
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.77688190192204
      },
      "nl": {
       "accuracy": 0.6,
       "cc": -1.4387554084615364,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": -4.316266225384613
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 10.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 10.483870967741936
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 75.04898636694963,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 72.7643945642429
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 75.04898636694963
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 68.3465266946212,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 60.00000000000001,
        "score": 66.25348697838334
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 68.3465266946212
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 55.426323731914415,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 57.3428009829773
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 55.426323731914415
      },
      "tr": {
       "accuracy": 1,
       "cc": 93.62057459269226,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 80.86172377807682
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 93.62057459269226
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.14352200896754,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 93.68863054303165
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.14352200896754
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 72.0478521212318,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 16.14355636369541
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 72.0478521212318
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.888,
       "cc": 85.53359229989094,
       "cc_by_type": {
        "choice": 85.53359229989094
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 85.53359229989094
      },
      "noul": {
       "accuracy": 0.8773333333333333,
       "cc": 75.46666666666665,
       "cc_by_type": {
        "noul": 75.46666666666665
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 75.46666666666665
      },
      "score": {
       "accuracy": 0.8186666666666667,
       "cc": 72.584235776724,
       "cc_by_type": {
        "score": 72.584235776724
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 72.584235776724
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 41.651717906240826,
       "cc_by_type": {
        "choice": 36,
        "noul": 50,
        "score": 38.95515371872248
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 41.651717906240826
      },
      "code_linting": {
       "accuracy": 1,
       "cc": 94.47310776421527,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 83.41932329264583
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 94.47310776421527
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 56.46158831917699,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 1.9050901607829962
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 56.46158831917699
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -3.5233882266319547,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": 16.33042011542578
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 42.42088485665628,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 75.0887415264906
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 42.42088485665628
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 36.769325748371706,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": 27.289109320586814
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 36.769325748371706
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 57.797958719568875,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 14.28571428571428,
        "score": 68.91208344161981
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 57.797958719568875
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 88.04718055411861,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 100,
        "score": 85.04746500730359
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 88.04718055411861
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 53.32086410018987,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 62.46259230056961
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 53.32086410018987
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 87.29303335849802,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 61.87910007549411
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 87.29303335849802
      },
      "lead_generation": {
       "accuracy": 0.88,
       "cc": 78.90939840172844,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 58.950417427407544
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 78.90939840172844
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 72.59080794067526,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 82.28855285428385
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 72.59080794067526
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 76.1961513748834,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 68.58845412465016
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 76.1961513748834
      },
      "model_routing": {
       "accuracy": 0.9170506912442397,
       "cc": 80.00471055982077,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 96.96969696969697,
        "score": 48.888590553921155
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.00471055982077
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 18.555142484275677,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 78.69805701136826
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 18.555142484275677
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 68.61731026200636,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 89.23168937469131
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 68.61731026200636
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 30.29288014362631,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 50,
        "score": -14.786384199662939
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 30.29288014362631
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 34.430741831633014,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 46.77048636446426
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 34.430741831633014
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.99953036759018,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.99859110277058
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.99953036759018
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      89.11942711171544,
      93.23983221561662
     ],
     "I": [
      72.92480724503656,
      80.87923205534871
     ],
     "capability": [
      81.53391121608728,
      86.64467671661372
     ],
     "composite": null,
     "seed": 43
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 92.98946060661117,
     "capability": 85.21346627565707,
     "intelligence": 77.43747194470296
    },
    "per_type": {
     "choice": {
      "accuracy": 0.888,
      "calibration": 94.29199900983717,
      "cc": 85.53359229989094,
      "cc_by_type": {
       "choice": 85.53359229989094
      },
      "cc_tiered": 83.72024973512157,
      "cc_tiered_open": 81.5623739827074,
      "cc_tiered_sealed": 85.87812548753575,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 85.53359229989094
     },
     "noul": {
      "accuracy": 0.8773333333333333,
      "calibration": 93.8354290070939,
      "cc": 75.46666666666665,
      "cc_by_type": {
       "noul": 75.46666666666665
      },
      "cc_tiered": 75.92501673005825,
      "cc_tiered_open": 79.59903381642512,
      "cc_tiered_sealed": 72.25099964369136,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 75.46666666666665
     },
     "score": {
      "accuracy": 0.8186666666666667,
      "calibration": 90.8409538029025,
      "cc": 72.584235776724,
      "cc_by_type": {
       "score": 72.584235776724
      },
      "cc_tiered": 72.66714936892907,
      "cc_tiered_open": 74.79070189113094,
      "cc_tiered_sealed": 70.54359684672721,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 72.584235776724
     }
    },
    "raw_on_A": {
     "calibration": 91.8267311771146,
     "capability": 84.43149101829664,
     "intelligence": 77.03625085947868
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 55.53718352397693,
    "Emphasis on Accuracy 60:20:20": 59.60108895026047,
    "Emphasis on Cost 20:20:60": 48.18065850796279,
    "Emphasis on Speed 20:60:20": 60.6633045064051,
    "Intelligence only": 66.94959306842838
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "PelaAI (Hugging Face)",
   "axes": {
    "calibration": 92.98322836048867,
    "cost": 63.558453547882905,
    "intelligence": 60.31798461023689,
    "speed": 91.72458132132627
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03408576592846662,
      "ece_score": 93.18284681430667,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 93.18284681430667,
      "score_before_time_limit": 93.18284681430667,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2453333333333333,
      "brier": 0.08846285900576352,
      "ece": 0.03670718036761854,
      "n": 375,
      "score": 92.65856392647629,
      "score_before_time_limit": 92.65856392647629,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9733333333333334,
      "interval90_width": 0.34444444444444444,
      "n": 375,
      "nrps": 0.06475366111989189,
      "score": 93.10827434068305,
      "score_before_time_limit": 93.10827434068305,
      "time_limit_failed_share": 0,
      "top_ece": 0.03654042603322353
     }
    },
    "score": 92.98322836048867
   },
   "capability": 76.65060648536277,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,528,836 input + 1,485 output tokens over 1,485 decisions = 1030.51919192 tokens/decision. GPU-time cost USD 0.0163952564839634/1,000; old list-price cost USD 0.322974747474747/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0163952564839634/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.565576875
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0163952564839634
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "KnowLine 35B-A3B-Gen1.5",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V. One VM for this row only (evaluator exposure and device ruling; --rows 35b-gen15). SGLang (127.0.0.1:9080) and author front end (127.0.0.1:8000) in ONE --network none --read-only container; driver requests serial over loopback. Read-only code, model and input, empty credential variables, HF offline flags; cut proof inside the container before sealed input (only loopback, no default route, DNS/TCP fail). Measured 23 Oct 2026 (reservation 08:42Z, VM created 08:43Z, teardown confirmed 08:58:23Z); measurement container 244 s (248 s including pulls), scored loop 122.727 s; run cap 2,219 s = 1.5 x the smoke-predicted 279 s plus 120 s per long item (15) under the corrected runner. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in container. Node-shred receipt verified; exact-id VM teardown confirms provider absent and guard closed. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 65.59226733572596,
    "I_sealed": 55.043701884747826,
    "base": 60.31798461023689,
    "excess": 0.7090386806224274,
    "gap": 10.548565450978138,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 78.95373257854448,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 75.46398238439761,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 55.547273982056595,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 62.275795446576836,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 76.61724391609044,
       "hard": 64.49987045709548,
       "judge": 55.519436791323265,
       "standard": 60.791459173663085
      }
     },
     "sealed|choice": {
      "cc": 74.63518188960879,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 88.56416772554002,
       "hard": 70.85694460506532,
       "judge": 72.67384916748287,
       "standard": 78.16916262391901
      }
     },
     "sealed|noul": {
      "cc": 37.1236894860158,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 32.72727272727274,
       "judge": 35.384615384615394,
       "standard": 34.32835820895524
      }
     },
     "sealed|score": {
      "cc": 53.37223427861889,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 67.89572537236607,
       "hard": 49.955408088998645,
       "judge": 48.9797607985226,
       "standard": 59.532851331130246
      }
     }
    }
   },
   "jevbench_score": 74.11637231061094,
   "key": "knowline-35b-a3b-gen1.5",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 weights (model card and LICENSE); MIT author serving code (file headers); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "PelaAI/KnowLine-35B-A3B-Gen1.5@9adcf717361b2f7d2946d093f1e85efdf1e59010",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.950530035335689,
    "decisive": 283,
    "decisive_rate": 0.7546666666666667,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 29,
   "ranked": true,
   "ranks": {
    "A": 29,
    "B": 47,
    "C": 25,
    "capability": 57
   },
   "repo": "https://huggingface.co/PelaAI/KnowLine-35B-A3B-Gen1.5",
   "scores": {
    "A": 74.11637231061094,
    "B": 70.87374598229188,
    "C": 74.11637231061094
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2451744479999661,
    "p50_s_raw": 0.04758722399998305,
    "p95_s_adjusted": 0.2741995019998285,
    "p95_s_raw": 0.06209975099991425
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER, run unmodified: PelaAI/KnowLine-35B-A3B-Gen1.5 at 9adcf717361b2f7d2946d093f1e85efdf1e59010, knowline_server.py sha256 ad922e7d2d717a6b9b936983ff5e5d69534edc593ed54b58732dc8ff60baa67e and knowline_layout.py sha256 aa0fb35619f4883edd6c3bd1cc6dc259f41f082cfe0471e586c60d783da2af9d, both mounted read-only at /model over stock SGLang 0.5.21. Only these two author Python files shipped; knowline_engine.py and serve_knowline.sh were excluded and never executed. Commands: env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONDONTWRITEBYTECODE=1 python3 -m sglang.launch_server --model-path /model --served-model-name m --tp 1 --quantization fp8 --mem-fraction-static 0.72 --mamba-radix-cache-strategy extra_buffer --enable-fp32-lm-head --host 127.0.0.1 --port 9080 ; env HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONDONTWRITEBYTECODE=1 python3 /model/knowline_server.py --model /model --backend sglang --url http://127.0.0.1:9080 --served-model-name knowline-35b-gen15 --temperature 1 --workers 16 --read-twice --twice-max-tokens 3000 --host 127.0.0.1 --port 8000. Image: lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469, derived image sha256:a829b8992c579540f1231a66ddd7732a470a22379d85a9f2138b6eb58bc1c187 (only blank-credential ENV, no package installs). Python 3.12.3, sglang 0.5.21, torch 2.13.0+cu130, transformers 5.12.1, tokenizers 0.22.2, flashinfer-python 0.6.18, sglang-kernel 0.4.7. Code gate: cleared by the benchmark owner on the disclosed patterns. DISCLOSED PATTERNS: dormant Hub-capable from_pretrained loaders in the unused --backend hf class; served tokenizer load from local /model under offline flags; HTTP client exclusively to loopback SGLang /generate. The source and runner were reviewed, with findings resolved in the amendments; the runner cap fix of Amendment 1c was independently reviewed (PASS) before this pod. Weights: Apache-2.0, public, ungated, 71,903,879,336 weight bytes, hash-checked file by file; pre-registration manifest sha256 61c753a9be64d055d16d69f2efa10b4346331844a447ed9799cd41357e38adab; byte-identical to Gen1 weights (revision e800968d). FIRST MEASUREMENT of this .5 serving version (our own intake). SAME WEIGHTS AS Gen1 (no earlier version of this exact model is on the board): the shards, tokenizer, configuration and chat template are byte-identical to the predecessor; the new author layout reads short text inputs twice, so this measures a new serving version, not new weights. AUTHOR'S OWN SERVER on our rented hardware, online FP8. NATIVE OUTPUTS: label-token logprobs, label-only softmax at temperature 1; choice argmax, Noul P(true), score expectation; header off, no rotation or calibration file. AUTHOR EXPOSURE (card, self-reported, not verified): the authors adopted read twice after checking their own evaluation sets and a fresh full run of the Decision Index 0.3 public suite; the card also reports JevBench / JevBench-hard evaluations. Training-data, Decision Index format-data and decontamination disclosures are inherited from Gen1; no overlap or contamination audit was run by us. The authors say no Jev or other decision-model outputs were used as training labels. The author's H20 timing does not transfer to this RTX PRO 6000 measurement. 10 s RULE: 0 decisions over 10 s, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); maximum 3.397 s (median 0.0476 s, p95 0.0621 s); the slowest decisions are the standard very long items of about 80,000 tokens. DETERMINISM: first 50 positions run twice, 2/50 bit-identical, 1 argmax flip, maximum probability difference 0.0395585 (the pre-registration expected 0 flips: a prediction miss; the scored raw is the first pass, so scores are unaffected; the 4B and 26B rows had 0 flips); bit reproducibility is not established and the cause was not investigated. CONTEXT: SGLang configuration limit 262,144 tokens; no author-imposed lower cap, no truncation and 0 refusals; the longest item was 80,522 tokens. SEPARATE POD: this row was measured alone on its own pod after the first pod (4B and 26B rows) ended with the 26B row flagged partial by the runner (its data are complete); the 35B exposure use was unspent at that point, the runner cap was corrected (Amendment 1c) and independently reviewed, and the run completed with complete=true, no deadline hit and rc 0 (run cap 2,219 s, scored loop 122.7 s). Over the cost cap under the earlier list-price basis; inside both caps from v1.8.0 (GPU-time cost basis).",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 78.53724442653552,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 43.94506661293989
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 78.53724442653552
      },
      "calibration_gen": {
       "accuracy": 0.5643564356435643,
       "cc": 37.12965075982285,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": 19.999999999999996,
        "score": 42.290330858471734
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 37.12965075982285
      },
      "composite_gen": {
       "accuracy": 0.8712871287128713,
       "cc": 78.59353672698349,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 70.37037037037037,
        "score": 71.9143048512305
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 78.59353672698349
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 95.05631152521637,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 85.1689345756491
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 95.05631152521637
      },
      "jaggedness": {
       "accuracy": 0.5944444444444444,
       "cc": 32.52691423486501,
       "cc_by_type": {
        "choice": 56.689124157844084,
        "noul": -8.108108108108103,
        "score": 48.99972665485905
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 32.52691423486501
      },
      "llm_router": {
       "accuracy": 0.865814696485623,
       "cc": 75.68588856509247,
       "cc_by_type": {
        "choice": 81.77335563969358,
        "noul": 85.56701030927836,
        "score": 59.717299746305486
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 75.68588856509247
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": -34.50220911933348,
       "cc_by_type": {
        "choice": 22.58064516129032,
        "noul": -100,
        "score": -26.08727251929075
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 27.769786201064957,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -25,
        "score": 57.79389468566909
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 27.769786201064957
      },
      "multilingual": {
       "accuracy": 0.7236363636363636,
       "cc": 49.53498435009352,
       "cc_by_type": {
        "choice": 72.61167264427779,
        "noul": 16.666666666666675,
        "score": 59.32661373933609
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 49.53498435009352
      },
      "negation_pair_gen": {
       "accuracy": 0.7906976744186046,
       "cc": 55.270855701977574,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 26.66666666666666,
        "score": 45.81256710593274
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 55.270855701977574
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 84.2589451817055,
       "cc_by_type": {
        "choice": 90.99099099099098,
        "noul": 100,
        "score": 61.785844554125546
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 84.2589451817055
      },
      "usecase": {
       "accuracy": 0.7090909090909091,
       "cc": 51.4472003671922,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 29.166666666666675,
        "score": 50.91022855255696
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 51.4472003671922
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 43.81520765690724,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": 38.58848011357887
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 43.81520765690724
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 25.42097445128751,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 30.33358387613595
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 25.42097445128751
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 92.14511885715005,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 92.36456011127314
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 92.14511885715005
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 44.1730779469375,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": 0,
        "score": 99.55833998606387
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.1730779469375
      },
      "en": {
       "accuracy": 0.7654213727193745,
       "cc": 61.10073141265881,
       "cc_by_type": {
        "choice": 77.59363444429528,
        "noul": 49.64028776978417,
        "score": 56.06827202389697
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 61.10073141265881
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 71.25620747667905,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 23.556550651896877
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.25620747667905
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 53.099368633050915,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 91.53854305762268
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 53.099368633050915
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 80.39360734662309,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 83.53376321633982
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 80.39360734662309
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 41.58116161609464,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 0,
        "score": 41.50326138459677
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 41.58116161609464
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 75.32642366956557,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 40.00888909131558
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 75.32642366956557
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 14.49636466714197,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 0,
        "score": 0.6773386400279802
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 14.49636466714197
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -1.8225924303067036,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 38.976667153524346
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.22713475233114,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 98.34807092366006
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.22713475233114
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 3.7864891493847828,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 11.359467448154348
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 3.7864891493847828
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 50.02576937954863,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 0,
        "score": 65.31217906543388
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 50.02576937954863
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 59.34634800662246,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 60.46685780890688
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 59.34634800662246
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 40.91896828063803,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 79.35264952276515
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 40.91896828063803
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 47.842968553972014,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": -6.471094338083971
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 47.842968553972014
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 73.03295672336854,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 83.6149992023637
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 73.03295672336854
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 49.64993222925987,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 15.616463354446298
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 49.64993222925987
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8146666666666667,
       "cc": 76.06153963910522,
       "cc_by_type": {
        "choice": 76.06153963910522
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 76.06153963910522
      },
      "noul": {
       "accuracy": 0.7173333333333334,
       "cc": 43.466666666666676,
       "cc_by_type": {
        "noul": 43.466666666666676
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 43.466666666666676
      },
      "score": {
       "accuracy": 0.68,
       "cc": 56.30595230792736,
       "cc_by_type": {
        "score": 56.30595230792736
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 56.30595230792736
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 14.797508704824592,
       "cc_by_type": {
        "choice": 52,
        "noul": 0,
        "score": -7.607473885526228
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 14.797508704824592
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 58.11375223901799,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 55.55555555555556,
        "score": 37.419241534169224
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 58.11375223901799
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 35.666240626404736,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50,
        "score": -26.741115519159784
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 35.666240626404736
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -27.686685872344828,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -12.884619020543253
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 62.73448675841547,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 83.8556341882899
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 62.73448675841547
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -15.37956527962207,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 0,
        "score": -83.24561407785993
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 18.904770744015718,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -42.85714285714286,
        "score": 38.78714136369983
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 18.904770744015718
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 47.56112704589649,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 0,
        "score": 63.58930448263722
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 47.56112704589649
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 40.756925480947764,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 83.93744310950997
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 40.756925480947764
      },
      "knowledge_graphs": {
       "accuracy": 0.7727272727272727,
       "cc": 69.18952787494685,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 53.74988269693192
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 69.18952787494685
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 43.90782621530675,
       "cc_by_type": {
        "choice": 100,
        "noul": 11.111111111111116,
        "score": 20.612367534809117
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 43.90782621530675
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -27.40129750584028,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": -19.838301119671375
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 66.25297917029879,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 50.27525229016698
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 66.25297917029879
      },
      "model_routing": {
       "accuracy": 0.8525345622119815,
       "cc": 72.27971344822836,
       "cc_by_type": {
        "choice": 83.63636363636363,
        "noul": 84.84848484848484,
        "score": 48.35429185983661
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 72.27971344822836
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 13.455828237937178,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 74.91642905162344
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 13.455828237937178
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 36.11315723953092,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": -33.333333333333336,
        "score": 58.385896973931665
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 36.11315723953092
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 40.47435635108647,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": 50.97971929956482
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 40.47435635108647
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 13.82491470436007,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -66.66666666666667,
        "score": 66.1124252725005
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 13.82491470436007
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.78505156447903,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.35515469343709
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.78505156447903
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.00685715125017,
      92.14901128901427
     ],
     "I": [
      55.14812372594048,
      64.35103732240668
     ],
     "capability": [
      71.82312622073084,
      77.47359939236085
     ],
     "composite": null,
     "seed": 43
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 92.98322836048867,
     "capability": 76.65060648536277,
     "intelligence": 60.31798461023689
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8146666666666667,
      "calibration": 93.18284681430667,
      "cc": 76.06153963910522,
      "cc_by_type": {
       "choice": 76.06153963910522
      },
      "cc_tiered": 76.79445723407663,
      "cc_tiered_open": 78.95373257854448,
      "cc_tiered_sealed": 74.63518188960879,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.06153963910522
     },
     "noul": {
      "accuracy": 0.7173333333333334,
      "calibration": 92.65856392647629,
      "cc": 43.466666666666676,
      "cc_by_type": {
       "noul": 43.466666666666676
      },
      "cc_tiered": 46.33548173403619,
      "cc_tiered_open": 55.547273982056595,
      "cc_tiered_sealed": 37.1236894860158,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 43.466666666666676
     },
     "score": {
      "accuracy": 0.68,
      "calibration": 93.10827434068305,
      "cc": 56.30595230792736,
      "cc_by_type": {
       "score": 56.30595230792736
      },
      "cc_tiered": 57.82401486259786,
      "cc_tiered_open": 62.275795446576836,
      "cc_tiered_sealed": 53.37223427861889,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 56.30595230792736
     }
    },
    "raw_on_A": {
     "calibration": 89.49061199462261,
     "capability": 75.80512414699561,
     "intelligence": 62.11963629936862
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.42105578449919,
    "Emphasis on Accuracy 60:20:20": 65.46888200769634,
    "Emphasis on Cost 20:20:60": 66.95084967264951,
    "Emphasis on Speed 20:60:20": 76.90064512771335,
    "Intelligence only": 60.31798461023689
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "PelaAI (Hugging Face)",
   "axes": {
    "calibration": 85.97018847365896,
    "cost": 69.83660860914662,
    "intelligence": 46.410576498288506,
    "speed": 92.9847804332849
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05901810910722855,
      "ece_score": 88.1963781785543,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 88.1963781785543,
      "score_before_time_limit": 88.1963781785543,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.29600000000000004,
      "brier": 0.14439865182207542,
      "ece": 0.0905954838815404,
      "n": 375,
      "score": 81.88090322369193,
      "score_before_time_limit": 81.88090322369193,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9306666666666666,
      "interval90_width": 0.3502222222222223,
      "n": 375,
      "nrps": 0.08820200493481374,
      "score": 87.8332840187307,
      "score_before_time_limit": 87.8332840187307,
      "time_limit_failed_share": 0,
      "top_ece": 0.0775661573452862
     }
    },
    "score": 85.97018847365896
   },
   "capability": 66.19038248597374,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,528,836 input + 1,485 output tokens over 1,485 decisions = 1030.51919192 tokens/decision. GPU-time cost USD 0.0101261971768405/1,000; old list-price cost USD 0.0310355757575758/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0101261971768405/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05432537999999999
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.010126197176840546
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of this serving version. SAME WEIGHTS AS Gen5 (no earlier version of this exact model is on the board): the shards, tokenizer, configuration and chat template are byte-identical to the predecessor; the new author layout reads short text inputs twice, so this measures a new serving version, not new weights. AUTHOR'S OWN SERVER on our rented hardware, online FP8. NATIVE OUTPUTS: label-token logprobs, label-only softmax at temperature 1; choice argmax, Noul P(true), score expectation; header off, no rotation or calibration file. AUTHOR EXPOSURE (card, self-reported, not verified): the authors adopted read twice after checking their own evaluation sets and a fresh full run of the Decision Index 0.3 public suite; the card also reports JevBench / JevBench-hard evaluations. Training-data, Decision Index format-data and decontamination disclosures are inherited from Gen5; no overlap or contamination audit was run by us. The authors say no Jev or other decision-model outputs were used as training labels. The author's H20 timing does not transfer to this RTX PRO 6000 measurement. 10 s RULE: 0 decisions over 10 s, maximum 2.958 s (median 0.0299 s, p95 0.0449 s). DETERMINISM: first 50 positions run twice, 6/50 bit-identical, 0 argmax flips, maximum probability difference 0.0583609; bit reproducibility is not established and the cause was not investigated. CONTEXT: SGLang configuration limit 262,144 tokens; no author-imposed lower cap, no truncation and 0 refusals; the longest item was 80,522 tokens. The answer file was lost on 26 Oct 2026 after scoring. These are the scores and confidence intervals computed on 23 Oct; the measurement cannot be scored again. Subject-topic and use-case breakdowns are unavailable.",
   "display": "KnowLine 4B-Gen5.5",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.18",
    "results_sha256": "387cbe05e0ed624c6fb19a4812a7abee9a02b65b559170c316faaadbf957e7cf",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Self-hosted on one evaluator-rented NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB GPU in Finland (EU), using the author's own server over SGLang 0.5.21. Contributor code ran in an offline container on a disposable rented GPU VM. Measured 23 Oct 2026. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 48.97603506848304,
    "I_sealed": 43.84511792809398,
    "base": 46.410576498288506,
    "excess": -4.708609629966645,
    "gap": 5.130917140389066,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 72.09931450776475,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 69.80182447310474,
       "judge": 76.26895187870798,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 27.249137336093852,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 13.043478260869556,
       "judge": 19.999999999999996,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 47.57965336159056,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.32825935892299,
       "hard": 49.16192766350832,
       "judge": 52.42653625372016,
       "standard": 34.77047742089441
      }
     },
     "sealed|choice": {
      "cc": 67.28758499042851,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 80.94027954256671,
       "hard": 62.530357349369716,
       "judge": 65.62193927522038,
       "standard": 72.47416156928918
      }
     },
     "sealed|noul": {
      "cc": 19.791203135516056,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 41.37931034482758,
       "hard": 19.999999999999996,
       "judge": 4.615384615384621,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 44.45656565833736,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 67.88669956759101,
       "hard": 41.1343549949148,
       "judge": 36.34395939289203,
       "standard": 51.55482942872366
      }
     }
    }
   },
   "jevbench_score": 59.16168733982439,
   "key": "knowline-4b-gen5.5",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 weights (model card and LICENSE); MIT author serving code (file headers); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.18",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": {
    "base": "Qwen/Qwen3.5-4B",
    "model": "PelaAI/KnowLine-4B-Gen5.5",
    "revision": "779944d2cc8ffa19cef386aee83c83bcb967ade2"
   },
   "new_in": "v1.7.18",
   "noul_decisive": {
    "acc_among_decisive": 0.8901515151515151,
    "decisive": 264,
    "decisive_rate": 0.704,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "provider": "self-hosted",
   "rank": 112,
   "ranked": true,
   "ranks": {
    "A": 112,
    "B": 112,
    "C": 116,
    "capability": 124
   },
   "repo": "https://huggingface.co/PelaAI/KnowLine-4B-Gen5.5",
   "scores": {
    "A": 59.16168733982439,
    "B": 53.984073896375854,
    "C": 41.08450509710026
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.209745888000009,
    "p50_s_raw": 0.02987294400000451,
    "p95_s_adjusted": 0.2397885830000973,
    "p95_s_raw": 0.044894291500048665
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 32.271483849061006,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100,
        "score": -61.518881786150324
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 32.271483849061006
      },
      "calibration_gen": {
       "accuracy": 0.5445544554455446,
       "cc": 27.4782972298076,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": 12.00000000000001,
        "score": 30.880278751989053
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 27.4782972298076
      },
      "composite_gen": {
       "accuracy": 0.7623762376237624,
       "cc": 54.55565476371459,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 11.111111111111116,
        "score": 62.31195074100827
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 54.55565476371459
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 66.14310450329872,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 78.42931350989615
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 66.14310450329872
      },
      "jaggedness": {
       "accuracy": 0.5277777777777778,
       "cc": 20.085631482343643,
       "cc_by_type": {
        "choice": 39.364773820981725,
        "noul": -24.32432432432432,
        "score": 45.21644495037352
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 20.085631482343643
      },
      "llm_router": {
       "accuracy": 0.8146964856230032,
       "cc": 67.36698705015681,
       "cc_by_type": {
        "choice": 76.22611605177423,
        "noul": 71.13402061855669,
        "score": 54.74082448013951
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 67.36698705015681
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": -33.80675424843211,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -36.90413371303825
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": -4.842406559032431,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": -75,
        "score": -2.41381761524162
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6654545454545454,
       "cc": 38.54154804739285,
       "cc_by_type": {
        "choice": 68.6990544506032,
        "noul": 0,
        "score": 46.925589691575354
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 38.54154804739285
      },
      "negation_pair_gen": {
       "accuracy": 0.813953488372093,
       "cc": 52.334083287022224,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 23.668916527733373
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 52.334083287022224
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 85.36513627447022,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 100,
        "score": 74.11342684142865
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.36513627447022
      },
      "usecase": {
       "accuracy": 0.6045454545454545,
       "cc": 40.26392710502498,
       "cc_by_type": {
        "choice": 58.33333333333336,
        "noul": 20.833333333333325,
        "score": 41.62511464840826
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 40.26392710502498
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 2.340865742229314,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 0,
        "score": -54.88216467807396
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 2.340865742229314
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 31.90275099199359,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": 3.849574020527635
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 31.90275099199359
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 75.76883467403283,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 75,
        "score": 92.12951287165603
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 75.76883467403283
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 33.2080350682951,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 99.81032494417768
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 33.2080350682951
      },
      "en": {
       "accuracy": 0.7028670721112077,
       "cc": 49.32589986114521,
       "cc_by_type": {
        "choice": 68.80682442245028,
        "noul": 30.935251798561158,
        "score": 48.235623362424185
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 49.32589986114521
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 64.17846354246149,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 2.323318849244149
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 64.17846354246149
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 29.378301436600594,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 54.255122889036755
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 29.378301436600594
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 64.14090868183332,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 55.95213781020585
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 64.14090868183332
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 31.34390732478197,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -25,
        "score": 35.79149851065875
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 31.34390732478197
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 1,
       "cc": 91.80896175102725,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 75.42688525308174
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 91.80896175102725
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 14.871421496751667,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 0,
        "score": 1.8025091288570771
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 14.871421496751667
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 14.06772974175999,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 42.20318922527997
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 14.06772974175999
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.72798269411503,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.85061474901177
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.72798269411503
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 12.952751400870142,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 38.85825420261042
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 12.952751400870142
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -14.516129032258064,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 27.30534802450514,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -40,
        "score": 37.1509150003034
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 27.30534802450514
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 46.490121564866435,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 19.999999999999996,
        "score": 51.29122503637893
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 46.490121564866435
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 25.058552297049115,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 71.7714015719984
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 25.058552297049115
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 52.119232714688515,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 6.357698144065549
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 52.119232714688515
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 68.81427220395923,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 38.70088112800675
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 68.81427220395923
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 27.594745740513915,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": 0.29836151532706134
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 27.594745740513915
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.76,
       "cc": 69.0005549283377,
       "cc_by_type": {
        "choice": 69.0005549283377
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 69.0005549283377
      },
      "noul": {
       "accuracy": 0.6266666666666667,
       "cc": 25.333333333333343,
       "cc_by_type": {
        "noul": 25.333333333333343
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 25.333333333333343
      },
      "score": {
       "accuracy": 0.6373333333333333,
       "cc": 46.68788062047984,
       "cc_by_type": {
        "score": 46.68788062047984
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 46.68788062047984
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 21.86136133810103,
       "cc_by_type": {
        "choice": 52,
        "noul": 0,
        "score": 13.584084014303087
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 21.86136133810103
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 25.6154537153356,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 11.111111111111116,
        "score": -15.631209592433514
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 25.6154537153356
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 43.56653867453104,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": 63.21994122684514
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 43.56653867453104
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -25.44088045543351,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -6.14720276980929
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 23.479309036688374,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 47.24952131296367
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 23.479309036688374
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -24.043449996253656,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -33.333333333333336,
        "score": -88.4825512466226
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 36.82308774249817,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 14.28571428571428,
        "score": 35.39923521629005
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 36.82308774249817
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": 9.280235636167426,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -50,
        "score": 30.105515270871607
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 9.280235636167426
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 9.475501516136621,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -19.999999999999996,
        "score": 31.75983788174319
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 9.475501516136621
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 16.44712824676007,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 0,
        "score": 26.429179172828388
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 16.44712824676007
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 30.314030901538743,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 13.68111079247154
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 30.314030901538743
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -1.775411642170127,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": 24.78129195521005
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 43.968248197373384,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -19.999999999999996,
        "score": 63.4210593713908
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 43.968248197373384
      },
      "model_routing": {
       "accuracy": 0.847926267281106,
       "cc": 69.42197449390987,
       "cc_by_type": {
        "choice": 85.97402597402598,
        "noul": 81.81818181818181,
        "score": 40.47371568952184
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 69.42197449390987
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 13.45072149125923,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 74.90110881158958
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 13.45072149125923
      },
      "recruiting": {
       "accuracy": 0.5555555555555556,
       "cc": 44.2692158594779,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 33.33333333333333,
        "score": 49.61359001111711
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 44.2692158594779
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 35.618665694228405,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": 21.63432220583793
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 35.618665694228405
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": -1.8808128994518825,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -66.66666666666667,
        "score": 33.487996084253055
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.32309721478084,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 94.96929164434255
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.32309721478084
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.82696355138368,
      87.4429186707541
     ],
     "I": [
      40.19398743049258,
      51.42848868757998
     ],
     "capability": [
      61.45056151982998,
      68.62052143325377
     ],
     "composite": null,
     "seed": 44
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.97018847365896,
     "capability": 66.19038248597374,
     "intelligence": 46.410576498288506
    },
    "per_type": {
     "choice": {
      "accuracy": 0.76,
      "calibration": 88.1963781785543,
      "cc": 69.0005549283377,
      "cc_by_type": {
       "choice": 69.0005549283377
      },
      "cc_tiered": 69.69344974909663,
      "cc_tiered_open": 72.09931450776475,
      "cc_tiered_sealed": 67.28758499042851,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.0005549283377
     },
     "noul": {
      "accuracy": 0.6266666666666667,
      "calibration": 81.88090322369193,
      "cc": 25.333333333333343,
      "cc_by_type": {
       "noul": 25.333333333333343
      },
      "cc_tiered": 23.520170235804954,
      "cc_tiered_open": 27.249137336093852,
      "cc_tiered_sealed": 19.791203135516056,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 25.333333333333343
     },
     "score": {
      "accuracy": 0.6373333333333333,
      "calibration": 87.8332840187307,
      "cc": 46.68788062047984,
      "cc_by_type": {
       "score": 46.68788062047984
      },
      "cc_tiered": 46.01810950996396,
      "cc_tiered_open": 47.57965336159056,
      "cc_tiered_sealed": 44.45656565833736,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 46.68788062047984
     }
    },
    "raw_on_A": {
     "calibration": 85.71172015865717,
     "capability": 68.03803428468126,
     "intelligence": 50.36434841070533
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 55.44203877277494,
    "Emphasis on Accuracy 60:20:20": 48.01794680199904,
    "Emphasis on Cost 20:20:60": 57.24101996306071,
    "Emphasis on Speed 20:60:20": 63.23099693942221,
    "Intelligence only": 39.98626876227327
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "guru0381",
   "axes": {
    "calibration": 62.82741496733333,
    "cost": 84.1923853867817,
    "intelligence": 21.99889248592056,
    "speed": 90.96816394625492
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06249555555555558,
      "ece_score": 87.50088888888888,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.50088888888888,
      "score_before_time_limit": 87.50088888888888,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.16533333333333333,
      "brier": 0.2664339086133331,
      "ece": 0.2359746666666666,
      "n": 375,
      "score": 52.805066666666676,
      "score_before_time_limit": 52.805066666666676,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.6186666666666667,
      "interval90_width": 0.2953333333333333,
      "n": 375,
      "nrps": 0.21633341307111098,
      "score": 48.17628934644445,
      "score_before_time_limit": 48.17628934644445,
      "time_limit_failed_share": 0,
      "top_ece": 0.4100704
     }
    },
    "score": 62.82741496733333
   },
   "capability": 42.413153726626945,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 662,682 input + 163,746 output tokens over 1,485 decisions = 556.517171717 tokens/decision. GPU-time cost USD 0.00336449964726631/1,000; old list-price cost USD 0.0199516767676768/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00336449964726631/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.03297833333333333
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0033644996472663143
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Krino-2B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container (--network none); author's `python3 -m krino.serve` (Kev @fe64b127 serve path, documented per-type temperature map choice 1.95 / noul 0.30 / score 0.50 from krino.json), KEV_TRUNCATE_STATES=1, reference kernels (KEV_FUSED=0: the documented pip install has no flash-linear-attention) paid evaluation (public). Kev architecture (LoRA r16 + pointer head) on Qwen/Qwen3.5-2B-Base @b1485b2f. Runtime: reviewed pytorch 2.11 base + Kev uv.lock closure (torch 2.8.0) installed offline from SHA-256-pinned wheels (PR #335). Kev reports usage.output_tokens as the token count of the serialized answer JSON (nothing is generated); priced as reported at the reference output rate.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 26.37590231932532,
    "I_sealed": 17.6218826525158,
    "base": 21.99889248592056,
    "excess": -1.0855071035461918,
    "gap": 8.75401966680952,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 36.29349492821419,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 49.40540540540541,
       "hard": 37.71626297577854,
       "judge": 32.76203032300594,
       "standard": 32.18920050230222
      }
     },
     "open|noul": {
      "cc": 21.265010351966872,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 13.043478260869556,
       "judge": -9.999999999999998,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 21.5692016777949,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.23363636363638,
       "hard": 9.25462962962964,
       "judge": 16.76983050847457,
       "standard": 38.56518518518517
      }
     },
     "sealed|choice": {
      "cc": 36.28277020740255,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 46.632782719186785,
       "hard": 30.785243437030175,
       "judge": 35.651322233104814,
       "standard": 43.049989453701755
      }
     },
     "sealed|noul": {
      "cc": 16.58287775014486,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": 9.090909090909083,
       "judge": 26.15384615384615,
       "standard": 13.432835820895516
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": -5.103771929824585,
       "hard": -28.450816901408405,
       "judge": -18.68588541666667,
       "standard": 12.30559090909088
      }
     }
    }
   },
   "jevbench_score": 9.191434964036928,
   "key": "krino-2b",
   "last_measured_on": "2026-10-12",
   "licence": "Apache-2.0 (adapter, head and code; Kev Apache-2.0; base Qwen3.5-2B-Base Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Guru0381/krino-2b@ff185abf129206174c67ebbe6058cab4cc9ccaed",
   "noul_decisive": {
    "acc_among_decisive": 0.7124600638977636,
    "decisive": 313,
    "decisive_rate": 0.8346666666666667,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 62.82741496733333,
     "cost": 61.00061799127391,
     "intelligence": 21.99889248592056,
     "speed": 91.85485240061512
    },
    "board_release": "v1.6.43",
    "capability": 42.413153726626945,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 109,
    "capability_rank": 137,
    "composite_ci95": {
     "A": [
      3.904931584579009,
      14.406086750913971
     ],
     "B": [
      3.0813923851653424,
      12.304710499013083
     ],
     "C": [
      2.711758044846534,
      10.004226910356923
     ]
    },
    "jevbench_score": 8.734255211215622,
    "measured_in": "v1.6.10",
    "method": "v1.6",
    "open_board_rank": 137,
    "rank": 176,
    "ranks": {
     "A": 176,
     "B": 177,
     "C": 177,
     "capability": 201
    },
    "scores": {
     "A": 8.734255211215622,
     "B": 7.217212347659793,
     "C": 6.065455007788626
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21046640197746455,
     "p50_s_raw": 0.03023320098873228,
     "p95_s_adjusted": 0.30997879329370326,
     "p95_s_raw": 0.07998939664685163
    },
    "views": {
     "Balanced 33:33:33": 7.984144388471859,
     "Emphasis on Accuracy 60:20:20": 5.914448339576854,
     "Emphasis on Cost 20:20:60": 9.172396825433118,
     "Emphasis on Speed 20:60:20": 10.241235033626667,
     "Intelligence only": 4.258556788204026
    }
   },
   "rank": 217,
   "ranked": true,
   "ranks": {
    "A": 217,
    "B": 218,
    "C": 218,
    "capability": 241
   },
   "repo": "https://huggingface.co/Guru0381/krino-2b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 9.191434964036928,
    "B": 7.462585231410012,
    "C": 6.382940947247866
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21348997401073574,
    "p50_s_raw": 0.031744987005367875,
    "p95_s_adjusted": 0.3748056430893484,
    "p95_s_raw": 0.11240282154467421
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.45454545454545453,
       "cc": -3.9711111111111044,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 50,
        "score": -78.57999999999998
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.504950495049505,
       "cc": 31.44464464033227,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": 52,
        "score": -6.764687500000011
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 31.44464464033227
      },
      "composite_gen": {
       "accuracy": 0.36633663366336633,
       "cc": -31.327821902578023,
       "cc_by_type": {
        "choice": 41.463414634146346,
        "noul": -11.111111111111116,
        "score": -124.3357692307693
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.6041666666666666,
       "cc": 48.12166666666667,
       "cc_by_type": {
        "choice": 100,
        "noul": 6.666666666666665,
        "score": 37.69833333333336
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 48.12166666666667
      },
      "jaggedness": {
       "accuracy": 0.35,
       "cc": 1.3896814773217478,
       "cc_by_type": {
        "choice": 17.70933589990377,
        "noul": -18.918918918918916,
        "score": 5.378627450980389
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 1.3896814773217478
      },
      "llm_router": {
       "accuracy": 0.670926517571885,
       "cc": 54.443170038415836,
       "cc_by_type": {
        "choice": 50.86730650700009,
        "noul": 64.94845360824742,
        "score": 47.51374999999999
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 54.443170038415836
      },
      "long_state": {
       "accuracy": 0.26666666666666666,
       "cc": -24.669193548387096,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": -100,
        "score": 16.315
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2222222222222222,
       "cc": -23.08969072164948,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -25,
        "score": -45.29999999999998
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5018181818181818,
       "cc": 15.536204017083938,
       "cc_by_type": {
        "choice": 39.3544179980437,
        "noul": 13.888888888888884,
        "score": -6.634694835680777
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 15.536204017083938
      },
      "negation_pair_gen": {
       "accuracy": 0.38372093023255816,
       "cc": -51.32922222222222,
       "cc_by_type": {
        "choice": 26.666666666666668,
        "noul": -13.33333333333333,
        "score": -167.321
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.375,
       "cc": 20.106524633821575,
       "cc_by_type": {
        "choice": 20.106524633821575
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 20.106524633821575
      },
      "perturb_pair": {
       "accuracy": 0.625,
       "cc": 45.63698198198199,
       "cc_by_type": {
        "choice": 45.94594594594596,
        "noul": 60.00000000000001,
        "score": 30.964999999999996
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 45.63698198198199
      },
      "usecase": {
       "accuracy": 0.41363636363636364,
       "cc": 5.069023006535965,
       "cc_by_type": {
        "choice": 31.372549019607874,
        "noul": -12.5,
        "score": -3.6654799999999765
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 5.069023006535965
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.1875,
       "cc": -33.39369963369963,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": -50,
        "score": -35.895384615384614
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.46153846153846156,
       "cc": -9.327886840757811,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": -73.91300000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.625,
       "cc": 45.03688341669872,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 50,
        "score": 32.89826086956522
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 45.03688341669872
      },
      "el": {
       "accuracy": 0.26666666666666666,
       "cc": 12.071002482929854,
       "cc_by_type": {
        "choice": -0.5586592178770929,
        "noul": -33.333333333333336,
        "score": 70.10499999999999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 12.071002482929854
      },
      "en": {
       "accuracy": 0.4778453518679409,
       "cc": 15.730797082087726,
       "cc_by_type": {
        "choice": 35.85628684053157,
        "noul": 17.266187050359715,
        "score": -5.93008264462811
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 15.730797082087726
      },
      "es": {
       "accuracy": 0.5483870967741935,
       "cc": 11.50404119886126,
       "cc_by_type": {
        "choice": 51.06035889070147,
        "noul": 75,
        "score": -91.54823529411769
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 11.50404119886126
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 4.108670309653917,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": -22.64666666666666
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 4.108670309653917
      },
      "fr": {
       "accuracy": 0.42857142857142855,
       "cc": 11.742827657378745,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0,
        "score": -1.2421052631578888
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 11.742827657378745
      },
      "hi": {
       "accuracy": 0.38095238095238093,
       "cc": -23.99129422718808,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": 0,
        "score": -88.175
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.8,
       "cc": 18.527619047619037,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -144.4171428571429
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 18.527619047619037
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 21.515655458476743,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": 39.99999999999999,
        "score": 10.329333333333345
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 21.515655458476743
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -12.233253968253969,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": 0,
        "score": -3.36642857142857
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.7857142857142857,
       "cc": 79.685,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 72.38833333333332
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 79.685
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 26.801093795093788,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": -0.9820000000000162
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 26.801093795093788
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 42.704345349985395,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 3.3479069767441416
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 42.704345349985395
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 21.2760213786505,
       "cc_by_type": {
        "choice": 4.537418974661164,
        "noul": 60.00000000000001,
        "score": -0.7093548387096638
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 21.2760213786505
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 4.250780141843975,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 34.88
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 4.250780141843975
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": 3.729202898550722,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50,
        "score": -73.59500000000001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 3.729202898550722
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 77.39897849462365,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 64.455
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 77.39897849462365
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 3.433069679849351,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -5.519999999999969
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 3.433069679849351
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.512,
       "cc": 36.96779502095333,
       "cc_by_type": {
        "choice": 36.96779502095333
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 36.96779502095333
      },
      "noul": {
       "accuracy": 0.5946666666666667,
       "cc": 18.933333333333337,
       "cc_by_type": {
        "noul": 18.933333333333337
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 18.933333333333337
      },
      "score": {
       "accuracy": 0.328,
       "cc": -6.174584837544961,
       "cc_by_type": {
        "score": -6.174584837544961
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -42.54725490196079,
       "cc_by_type": {
        "choice": 36,
        "noul": -50,
        "score": -113.64176470588237
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.3333333333333333,
       "cc": -12.739704039921422,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -33.333333333333336,
        "score": -11.718076923076893
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 55.12989159891598,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": -2.090000000000014
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 55.12989159891598
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 34.89788499025341,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 100,
        "score": -13.43500000000002
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 34.89788499025341
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": 0.4748007246376762,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": -7.271250000000018
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0.4748007246376762
      },
      "feature_extraction": {
       "accuracy": 0.15789473684210525,
       "cc": -116.71251572327047,
       "cc_by_type": {
        "choice": -13.207547169811319,
        "noul": -100,
        "score": -236.9300000000001
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 11.735613014436538,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -42.85714285714286,
        "score": 27.08358974358973
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 11.735613014436538
      },
      "gaming": {
       "accuracy": 0.18181818181818182,
       "cc": -12.115642074433163,
       "cc_by_type": {
        "choice": -14.98257839721254,
        "noul": -50,
        "score": 28.63565217391304
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 26.11814814814815,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 60.00000000000001,
        "score": 1.687777777777777
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 26.11814814814815
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": -3.2785867237687394,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 0,
        "score": -19.900000000000006
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 31.765082217524082,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 33.33333333333333,
        "score": -14.782272727272705
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 31.765082217524082
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 9.02948476702508,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": -41.728750000000026
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 9.02948476702508
      },
      "llm_guardrails": {
       "accuracy": 0.2962962962962963,
       "cc": -19.13207400298573,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -60,
        "score": -28.29833333333338
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.6728110599078341,
       "cc": 52.803651357323226,
       "cc_by_type": {
        "choice": 59.0909090909091,
        "noul": 60.606060606060595,
        "score": 38.713984374999995
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 52.803651357323226
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 16.1079067404313,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": 0,
        "score": 5.905294117647076
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 16.1079067404313
      },
      "recruiting": {
       "accuracy": 0.3333333333333333,
       "cc": -14.35318239173391,
       "cc_by_type": {
        "choice": 33.147632311977716,
        "noul": -33.333333333333336,
        "score": -42.87384615384611
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.2222222222222222,
       "cc": -30.597434560030916,
       "cc_by_type": {
        "choice": 11.330049261083742,
        "noul": -50,
        "score": -53.12235294117649
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": 11.78285507246376,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": 0,
        "score": 7.81233333333331
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 11.78285507246376
      },
      "search_retrieval": {
       "accuracy": 0.5416666666666666,
       "cc": 29.112966223387648,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": 0,
        "score": 19.367368421052632
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 29.112966223387648
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      56.134235634833225,
      65.29815028441786
     ],
     "I": [
      16.102596267666183,
      26.8828021176775
     ],
     "capability": [
      36.956661624305724,
      45.50971580905799
     ],
     "composite": null,
     "seed": 43
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 62.82741496733333,
     "capability": 42.413153726626945,
     "intelligence": 21.99889248592056
    },
    "per_type": {
     "choice": {
      "accuracy": 0.512,
      "calibration": 87.50088888888888,
      "cc": 36.96779502095333,
      "cc_by_type": {
       "choice": 36.96779502095333
      },
      "cc_tiered": 36.28813256780837,
      "cc_tiered_open": 36.29349492821419,
      "cc_tiered_sealed": 36.28277020740255,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 36.96779502095333
     },
     "noul": {
      "accuracy": 0.5946666666666667,
      "calibration": 52.805066666666676,
      "cc": 18.933333333333337,
      "cc_by_type": {
       "noul": 18.933333333333337
      },
      "cc_tiered": 18.923944051055866,
      "cc_tiered_open": 21.265010351966872,
      "cc_tiered_sealed": 16.58287775014486,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.933333333333337
     },
     "score": {
      "accuracy": 0.328,
      "calibration": 48.17628934644445,
      "cc": -6.174584837544961,
      "cc_by_type": {
       "score": -6.174584837544961
      },
      "cc_tiered": 10.78460083889745,
      "cc_tiered_open": 21.5692016777949,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 63.43795542912849,
     "capability": 42.556073742724294,
     "intelligence": 21.674192056320095
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 8.499413197630872,
    "Emphasis on Accuracy 60:20:20": 6.078227628905059,
    "Emphasis on Cost 20:20:60": 10.511269154436356,
    "Emphasis on Speed 20:60:20": 10.717205035733256,
    "Intelligence only": 4.258556788204026
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Autoloops",
   "axes": {
    "calibration": 87.31,
    "cost": 40.47,
    "intelligence": 58.1,
    "speed": 84.44
   },
   "calibration": {
    "score": 87.31
   },
   "capability": 72.7,
   "class": "jev",
   "composite_ci95": {
    "A": [
     38.2,
     41.77
    ],
    "B": [
     37.03,
     42.17
    ],
    "C": [
     26.76,
     41.77
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.2/M input, 0.45/M output (spec kushal-gemma4-31b-it-autoloops.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.09648663282571913
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Autoloops – Gemma 4 31B IT",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 57.42,
    "I_sealed": 53.63,
    "base": 58.1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.1396,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.1081,
       "hard": 71.6892,
       "judge": 76.269,
       "standard": 69.8619
      }
     },
     "open|noul": {
      "cc": 50.0193,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 47.8261,
       "judge": 40,
       "standard": 44.4444
      }
     },
     "open|score": {
      "cc": 47.1011,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.1922,
       "hard": 49.5059,
       "judge": 45.7484,
       "standard": 34.2749
      }
     },
     "sealed|choice": {
      "cc": 80.2748,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 88.8786,
       "hard": 78.6707,
       "judge": 81.9095,
       "standard": 76.7292
      }
     },
     "sealed|noul": {
      "cc": 39.9198,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 18.1818,
       "judge": 50,
       "standard": 50
      }
     },
     "sealed|score": {
      "cc": 40.6891,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 81.6366,
       "hard": 8.7267,
       "judge": 56.0954,
       "standard": 61.0306
      }
     }
    }
   },
   "jevbench_score": 40.17,
   "key": "kushal-gemma4-31b-it-autoloops",
   "last_measured_on": "2026-10-06",
   "licence": "Gemma terms of use (Google) for the base weights; hosted Autoloops service",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 29,
    "axes": {
     "calibration": 87.31,
     "cost": 40.47,
     "intelligence": 58.1,
     "speed": 84.51
    },
    "board_release": "v1.6.43",
    "capability": 72.7,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      38.21,
      41.78
     ],
     "B": [
      37.03,
      42.18
     ],
     "C": [
      26.77,
      41.78
     ]
    },
    "jevbench_score": 40.18,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 82,
    "ranks": {
     "A": 82,
     "B": 76,
     "C": 70,
     "capability": 58
    },
    "scores": {
     "A": 40.18,
     "B": 39.73,
     "C": 37.67
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.584,
     "p50_s_raw": 0.584,
     "p95_s_adjusted": 0.606,
     "p95_s_raw": 0.606
    },
    "views": null
   },
   "rank": 147,
   "ranked": true,
   "ranks": {
    "A": 147,
    "B": 138,
    "C": 124,
    "capability": 77
   },
   "repo": "https://benchmarkheaven.com/jev-models/kushal-gemma4-31b-it-autoloops",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 40.17,
    "B": 39.73,
    "C": 37.66
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.584,
    "p50_s_raw": 0.584,
    "p95_s_adjusted": 0.616,
    "p95_s_raw": 0.616
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "18ef06f9a56b5d3e358f5a8ddd0bf942facf1bae52fb8b57b5336b8026c1be5f"
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Launchables",
   "axes": {
    "calibration": 84.04872871623093,
    "cost": 99.35683144033557,
    "intelligence": 57.25409127121432,
    "speed": 85.292646608578
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.058081079642959894,
      "ece_score": 88.38378407140802,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 88.38378407140802,
      "score_before_time_limit": 88.38378407140802,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.15128528819337783,
      "ece": 0.10546966253208946,
      "n": 375,
      "score": 78.90606749358211,
      "score_before_time_limit": 78.90606749358211,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.944,
      "interval90_width": 0.3946666666666667,
      "n": 375,
      "nrps": 0.08944470329719607,
      "score": 84.85633458370268,
      "score_before_time_limit": 84.85633458370268,
      "time_limit_failed_share": 0,
      "top_ece": 0.10671430251437525
     }
    },
    "score": 84.04872871623093
   },
   "capability": 70.65140999372262,
   "composite_ci95": {
    "A": [
     74.67174426737074,
     80.110488383192
    ],
    "B": [
     68.63780034080798,
     75.44547169412674
    ],
    "C": [
     55.61664767547676,
     80.110488383192
    ]
   },
   "cost": {
    "basis": "Public launch list price, counted under interpretation I-1 (a manufacturer's standard, non-promotional list price counts from day 1): GBP 0.0015 per million input tokens, output free, from launchables.ai/docs/deploy/hosted fetched 26 Oct 2026 17:38 UTC. USD 0.0020020 per million input tokens from the ECB euro reference rates of 20 Oct 2026 (GBP 1 = USD 1.33468); prices exclude VAT. Provider-reported usage: 779,294 input / 1,485 output tokens on the common 1,485-item basis; 989,607 input / 1,500 output over all 1,500 items. Common basis USD 0.00105060 per 1,000 decisions; all-item basis USD 0.00132080. The tariff applied to the row's own provider-reported tokens; not a bill and not a quote. Saved copy of the vendor page sha256 956fa709. Price corrected in v1.7.9; earlier releases showed a same-base ESTIMATE of USD 0.0264 per 1,000.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0013207954759999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0010506037629629628
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Myles-G 12B (Launchables)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid public fast-lane; full S1200+P300 draw v1.6-regular-20261012, unchanged evaluator API runner from Sandy, customer API key; full-set exposure gate logged all sealed items before the first scored request. Native probabilities; latency has no API adjustment.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 60.47280030970968,
    "I_sealed": 54.035382232718966,
    "base": 57.25409127121432,
    "excess": -3.402108693364994,
    "gap": 6.437418076990717,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 71.8314160213165,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 71.6892104435357,
       "judge": 56.49307844429796,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 55.719806763285035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 39.99999999999999,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 53.867178144527514,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.40372091886416,
       "hard": 53.51144632562129,
       "judge": 50.74381636500394,
       "standard": 54.99541306445699
      }
     },
     "sealed|choice": {
      "cc": 65.72076424974752,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 84.75222363405337,
       "hard": 63.05076905285068,
       "judge": 60.33300685602351,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 56.64273760207883,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 49.09090909090909,
       "judge": 50.76923076923077,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 39.74264484633055,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 56.81348601357418,
       "hard": 39.23170405307833,
       "judge": 29.28973440649999,
       "standard": 47.908471508959
      }
     }
    }
   },
   "jevbench_score": 78.19688312813278,
   "key": "launchables-myles-g-12b",
   "last_measured_on": "2026-10-21",
   "licence": "Proprietary vendor-hosted API; weights and serving code not independently inspected",
   "listing": "ranked",
   "measured_in": "v1.6.38",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:https://www.launchables.ai/licensing#launchables-myles-g-12b",
   "noul_decisive": {
    "acc_among_decisive": 0.7946666666666666,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 18,
    "axes": {
     "calibration": 84.04872871623093,
     "cost": 57.35738348807602,
     "intelligence": 57.25409127121432,
     "speed": 85.64932567577961
    },
    "board_release": "v1.6.43",
    "capability": 70.65140999372262,
    "capability_api_board_rank": 18,
    "capability_open_board_rank": null,
    "capability_rank": 43,
    "composite_ci95": {
     "A": [
      65.69355393933306,
      69.86651967043076
     ],
     "B": [
      62.36968910887525,
      67.94030392837915
     ],
     "C": [
      48.91386791946196,
      69.86651967043076
     ]
    },
    "jevbench_score": 68.40656153142275,
    "measured_in": "v1.6.38",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 34,
    "ranks": {
     "A": 34,
     "B": 39,
     "C": 41,
     "capability": 74
    },
    "scores": {
     "A": 68.40656153142275,
     "B": 65.84152005408374,
     "C": 62.28856307389066
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.479910550522618,
     "p50_s_raw": 0.479910550522618,
     "p95_s_adjusted": 0.5674232601770199,
     "p95_s_raw": 0.5674232601770199
    },
    "views": {
     "Balanced 33:33:33": 64.4107683660651,
     "Emphasis on Accuracy 60:20:20": 61.34362333393462,
     "Emphasis on Cost 20:20:60": 61.39100451654052,
     "Emphasis on Speed 20:60:20": 71.50304365746292,
     "Intelligence only": 57.25409127121432
    }
   },
   "rank": 3,
   "ranked": true,
   "ranks": {
    "A": 3,
    "B": 31,
    "C": 53,
    "capability": 94
   },
   "repo": "https://www.launchables.ai/licensing",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 78.19688312813278,
    "B": 72.86618513011912,
    "C": 71.20327901104855
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4840246535022743,
    "p50_s_raw": 0.4840246535022743,
    "p95_s_adjusted": 0.6107561607146635,
    "p95_s_raw": 0.6107561607146635
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API through the unchanged TypeSafe adapter; no vendor code ran locally. TRUNCATION ON THE 15 LONGEST ITEMS (scored as answered): the provider silently truncated their input and reported 12,511 to 15,221 tokens; it returned answers, not refusals, under its 16,384-token context. The scored-wrong rule applies only to refusals, so these items are scored as answered, as for earlier provider-truncated rows. SAME-AUTHOR SECOND use after Myles-G 31B full-set exposure; the public set is treated as seen. Customer states merged LoRA on google/gemma-4-12B-it; deployed weights, serving and training claims are unverified. Measured accuracy on the 15 shortened states: 2 of 15 correct (13.3 %), versus 1083 of 1485 (72.9 %) on the remaining items; a Score item is correct within half a label step.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 73.02371545408378,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 44.071146362251355
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 73.02371545408378
      },
      "calibration_gen": {
       "accuracy": 0.6237623762376238,
       "cc": 55.34888443858599,
       "cc_by_type": {
        "choice": 42.73594909862143,
        "noul": 84.00000000000001,
        "score": 39.31070421713652
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 55.34888443858599
      },
      "composite_gen": {
       "accuracy": 0.7029702970297029,
       "cc": 41.59179534449972,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": 3.703703703703698,
        "score": 47.0879424923971
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 41.59179534449972
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 85.71737204456754,
       "cc_by_type": {
        "choice": 100,
        "noul": 86.66666666666667,
        "score": 70.48544946703595
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.71737204456754
      },
      "jaggedness": {
       "accuracy": 0.5666666666666667,
       "cc": 34.81576106951519,
       "cc_by_type": {
        "choice": 48.0269489894129,
        "noul": 40.54054054054055,
        "score": 15.879793678592124
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 34.81576106951519
      },
      "llm_router": {
       "accuracy": 0.8466453674121406,
       "cc": 75.82535881523177,
       "cc_by_type": {
        "choice": 76.22611605177423,
        "noul": 91.75257731958763,
        "score": 59.497383074333456
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 75.82535881523177
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -46.41598515353363,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100,
        "score": -36.022149008987995
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 41.50206138678643,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 50,
        "score": 23.990720242833497
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 41.50206138678643
      },
      "multilingual": {
       "accuracy": 0.7345454545454545,
       "cc": 60.13778089654668,
       "cc_by_type": {
        "choice": 68.6990544506032,
        "noul": 55.55555555555556,
        "score": 56.15873268348128
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 60.13778089654668
      },
      "negation_pair_gen": {
       "accuracy": 0.6627906976744186,
       "cc": 29.911992335055242,
       "cc_by_type": {
        "choice": 76.66666666666666,
        "noul": 0,
        "score": 13.069310338499063
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 29.911992335055242
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 74.00836593423553,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100,
        "score": 52.0551278327366
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 74.00836593423553
      },
      "usecase": {
       "accuracy": 0.6545454545454545,
       "cc": 52.849677358848375,
       "cc_by_type": {
        "choice": 63.23529411764707,
        "noul": 54.16666666666667,
        "score": 41.147071292231374
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 52.849677358848375
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": 3.819060228511955,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0,
        "score": -12.352343123987941
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 3.819060228511955
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 52.10545103048704,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 43.72034694706789
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 52.10545103048704
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 85.79316708994303,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 81.27330657956358
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 85.79316708994303
      },
      "el": {
       "accuracy": 0.6,
       "cc": 43.00860821915449,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -33.333333333333336,
        "score": 95.8787110634225
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 43.00860821915449
      },
      "en": {
       "accuracy": 0.7185056472632494,
       "cc": 55.84520341950083,
       "cc_by_type": {
        "choice": 68.36748392135803,
        "noul": 57.55395683453237,
        "score": 41.614169502612086
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 55.84520341950083
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 66.18505736183396,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 100,
        "score": 27.91895675108097
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.18505736183396
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 53.040130879712194,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 91.36082979760654
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 53.040130879712194
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 75.9551412851916,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 70.2183650320454
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 75.9551412851916
      },
      "hi": {
       "accuracy": 1,
       "cc": 81.96851164064229,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100,
        "score": 45.90553492192687
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 81.96851164064229
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 58.47867787426869,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 3.4952697880437866
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 58.47867787426869
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 47.9045972019261,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 60.00000000000001,
        "score": 40.90203624438037
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 47.9045972019261
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 48.894562335279964,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 46.6836870058399
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 48.894562335279964
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.62497577118171,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 93.54159398021181
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.62497577118171
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 36.93305405397785,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 29.413880776652157
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 36.93305405397785
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 59.45596416808655,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 53.60276343104764
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 59.45596416808655
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 73.31967475908746,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 62.38683806630188
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 73.31967475908746
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 27.84815255959744,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": 60.00000000000001,
        "score": 25.672117253260396
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 27.84815255959744
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 65.84902838886349,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 19.28621560137309
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 65.84902838886349
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 67.08568154736894,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 65.77317367436487
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.08568154736894
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 66.36968910397002,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -0.8909326880899515
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 66.36968910397002
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.752,
       "cc": 67.96724009261563,
       "cc_by_type": {
        "choice": 67.96724009261563
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 67.96724009261563
      },
      "noul": {
       "accuracy": 0.7946666666666666,
       "cc": 58.93333333333332,
       "cc_by_type": {
        "noul": 58.93333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 58.93333333333332
      },
      "score": {
       "accuracy": 0.5946666666666667,
       "cc": 44.11986046405637,
       "cc_by_type": {
        "score": 44.11986046405637
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 44.11986046405637
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 21.436138948141217,
       "cc_by_type": {
        "choice": 52,
        "noul": 0,
        "score": 12.308416844423641
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 21.436138948141217
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 59.65610142196325,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 77.77777777777777,
        "score": 19.824066860782764
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 59.65610142196325
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 44.27306472524267,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 15.339519378980015
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 44.27306472524267
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 18.13548546873989,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -8.751438330622419
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 18.13548546873989
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 22.251914893223415,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 29.07458525938037
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.251914893223415
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 15.587403181990245,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 66.66666666666667,
        "score": -69.5899917118909
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 15.587403181990245
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 38.185896214716806,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 14.28571428571428,
        "score": 39.48766063294595
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 38.185896214716806
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": 1.5120896906177588,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100,
        "score": 56.801077434222606
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 1.5120896906177588
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 52.701004610275824,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 100,
        "score": 20.603013830827464
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 52.701004610275824
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 59.81133967013818,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 100,
        "score": 30.825881965446655
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 59.81133967013818
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 48.49298180723741,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 1.5512968429008622
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 48.49298180723741
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 63.20796045838467,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 21.88194589128305
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 63.20796045838467
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 83.43877597847415,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 100,
        "score": 61.83264271469309
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 83.43877597847415
      },
      "model_routing": {
       "accuracy": 0.8571428571428571,
       "cc": 76.57582004023605,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 100,
        "score": 49.59758999083801
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 76.57582004023605
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 42.92899893712856,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 0,
        "score": 63.335941149197595
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 42.92899893712856
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.976710098847924,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 65.02298080722161
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.976710098847924
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 51.03200757320826,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50,
        "score": 32.65267296593021
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 51.03200757320826
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 23.916868816794192,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": 33.33333333333333,
        "score": 39.86654847936809
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 23.916868816794192
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 91.10806316661156,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100,
        "score": 94.67650266709447
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 91.10806316661156
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.91555558047827,
      85.87719178871096
     ],
     "I": [
      51.80648252445323,
      61.39756475428746
     ],
     "capability": [
      65.8773400775301,
      72.96432355292742
     ],
     "composite": {
      "A": [
       74.67174426737074,
       80.110488383192
      ],
      "B": [
       68.63780034080798,
       75.44547169412674
      ],
      "C": [
       55.61664767547676,
       80.110488383192
      ]
     },
     "seed": 46
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.04872871623093,
     "capability": 70.65140999372262,
     "intelligence": 57.25409127121432
    },
    "per_type": {
     "choice": {
      "accuracy": 0.752,
      "calibration": 88.38378407140802,
      "cc": 67.96724009261563,
      "cc_by_type": {
       "choice": 67.96724009261563
      },
      "cc_tiered": 68.776090135532,
      "cc_tiered_open": 71.8314160213165,
      "cc_tiered_sealed": 65.72076424974752,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.96724009261563
     },
     "noul": {
      "accuracy": 0.7946666666666666,
      "calibration": 78.90606749358211,
      "cc": 58.93333333333332,
      "cc_by_type": {
       "noul": 58.93333333333332
      },
      "cc_tiered": 56.18127218268194,
      "cc_tiered_open": 55.719806763285035,
      "cc_tiered_sealed": 56.64273760207883,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 58.93333333333332
     },
     "score": {
      "accuracy": 0.5946666666666667,
      "calibration": 84.85633458370268,
      "cc": 44.11986046405637,
      "cc_by_type": {
       "score": 44.11986046405637
      },
      "cc_tiered": 46.80491149542903,
      "cc_tiered_open": 53.867178144527514,
      "cc_tiered_sealed": 39.74264484633055,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 44.11986046405637
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 76.42324135863578,
    "Emphasis on Accuracy 60:20:20": 67.39718427116219,
    "Emphasis on Cost 20:20:60": 84.19699741255775,
    "Emphasis on Speed 20:60:20": 79.74004192517185,
    "Intelligence only": 57.25409127121432
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Launchables",
   "axes": {
    "calibration": 83.99531596546265,
    "cost": 51.11161516579885,
    "intelligence": 73.64186833978687,
    "speed": 85.94853930142175
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06430449201022093,
      "ece_score": 87.13910159795581,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.13910159795581,
      "score_before_time_limit": 87.13910159795581,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.08560855771666369,
      "ece": 0.06318301734560669,
      "n": 375,
      "score": 87.36339653087866,
      "score_before_time_limit": 87.36339653087866,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.824,
      "interval90_width": 0.114,
      "n": 375,
      "nrps": 0.07920606342697788,
      "score": 77.48344976755348,
      "score_before_time_limit": 77.48344976755348,
      "time_limit_failed_share": 0,
      "top_ece": 0.18556247061097633
     }
    },
    "score": 83.99531596546265
   },
   "capability": 78.81859215262476,
   "composite_ci95": {
    "A": [
     68.95161343706428,
     71.55692630209987
    ],
    "B": [
     68.92716284510078,
     72.62891518757193
    ],
    "C": [
     68.95161343706428,
     71.55692630209987
    ]
   },
   "cost": {
    "basis": "ESTIMATE, v1.5-M2 base-model reference rule. Launchables publishes a list price for Myles-G 31B (launchables.ai/licensing, fetched 21 Oct 2026, saved copy sha256 ad4736d5...): GBP 0.005 per million input tokens, output free; that is USD 0.0066734 per million input tokens at the ECB reference rate of 20 Oct 2026 (GBP 1 = USD 1.33468), prices exclude VAT. Under the method, the base-model reference price applies as an ESTIMATE for this vendor-hosted row, as for attest-31b-v1.0: base model google/gemma-4-31B-it through the frozen pricing_v15 module (sha256 1f4d1f7b1ce97ab27b938043444b9c2757e4a9dbecc799a803122588401f5bf1) reference() -> google/gemma-4-31b-it on OpenRouter, USD 0.09 per million input tokens, USD 0.34 per million output tokens. The vendor list price is shown for information and is not used for the Cost axis. Applied to this row's own provider-reported tokens (907,107 input and 1,500 output tokens over all 1,500 items; 697,619 input and 1,485 output tokens over the 1,485 items of the common cost basis; the 15 truncated items count for no row's cost). Estimated, not a vendor bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them in full).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05476641999999999
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0426199393939394
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Myles-G 31B (Launchables)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "paid evaluation. Vendor-hosted Launchables API (api.launchables.ai, model `myles-g-31b`, TypeSafe-compatible endpoint), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) over the official per-item latency subset without API adjustment; one non-item warm-up request preceded the scored loop and is excluded. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `launchables` (no proxy) before the first request. All 1,500 of 1,500 requests answered with HTTP 200 and the served model was `myles-g-31b` on every row; native probabilities on all three question types. Run with the customer's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 76.07886474383956,
    "I_sealed": 71.20487193573419,
    "base": 73.64186833978687,
    "excess": -4.9655339622503405,
    "gap": 4.8739928081053705,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 79.98663238745361,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 79.2387543252595,
       "judge": 72.31377719182598,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 87.52173913043478,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 70,
       "standard": 100
      }
     },
     "open|score": {
      "cc": 60.72822271363029,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 80.5682624678971,
       "hard": 65.56227672741873,
       "judge": 55.992001550320516,
       "standard": 48.24442655388464
      }
     },
     "sealed|choice": {
      "cc": 80.48409588326635,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 78.66312015727996,
       "judge": 77.96278158667972,
       "standard": 82.91499683611053
      }
     },
     "sealed|noul": {
      "cc": 75.46738696198295,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 67.27272727272727,
       "judge": 81.53846153846153,
       "standard": 79.10447761194031
      }
     },
     "sealed|score": {
      "cc": 57.66313296195325,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 77.78722486429184,
       "hard": 61.05229981148842,
       "judge": 48.908706111046754,
       "standard": 53.95439358807338
      }
     }
    }
   },
   "jevbench_score": 70.56596417573436,
   "key": "launchables-myles-g-31b",
   "last_measured_on": "2026-10-21",
   "licence": "Proprietary hosted API (no public weights release; the merged adapter is gated on Hugging Face); declared base google/gemma-4-31B-it is under the Gemma 4 licence",
   "listing": "ranked",
   "measured_in": "v1.6.36",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.launchables.ai/m/myles-g-31b#myles-g-31b",
   "noul_decisive": {
    "acc_among_decisive": 0.8933333333333333,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 15,
    "axes": {
     "calibration": 83.99531596546265,
     "cost": 51.11161516579885,
     "intelligence": 73.64186833978687,
     "speed": 86.60775159362743
    },
    "board_release": "v1.6.43",
    "capability": 78.81859215262476,
    "capability_api_board_rank": 8,
    "capability_open_board_rank": null,
    "capability_rank": 16,
    "composite_ci95": {
     "A": [
      69.05703328640422,
      71.67046972209141
     ],
     "B": [
      69.01141312494278,
      72.72246397695918
     ],
     "C": [
      69.05703328640422,
      71.67046972209141
     ]
    },
    "jevbench_score": 70.6763821133842,
    "measured_in": "v1.6.36",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 24,
    "ranks": {
     "A": 24,
     "B": 14,
     "C": 22,
     "capability": 31
    },
    "scores": {
     "A": 70.6763821133842,
     "B": 71.25021746899355,
     "C": 70.6763821133842
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.4372560594929382,
     "p50_s_raw": 0.4372560594929382,
     "p95_s_adjusted": 0.49944653444108555,
     "p95_s_raw": 0.49944653444108555
    },
    "views": {
     "Balanced 33:33:33": 67.12826022407477,
     "Emphasis on Accuracy 60:20:20": 69.59036149121279,
     "Emphasis on Cost 20:20:60": 59.65119627360718,
     "Emphasis on Speed 20:60:20": 73.7646036645773,
     "Intelligence only": 73.64186833978687
    }
   },
   "rank": 71,
   "ranked": true,
   "ranks": {
    "A": 71,
    "B": 43,
    "C": 55,
    "capability": 41
   },
   "repo": "https://www.launchables.ai/licensing",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.56596417573436,
    "B": 71.16041573983628,
    "C": 70.56596417573436
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4449399680015631,
    "p50_s_raw": 0.4449399680015631,
    "p95_s_adjusted": 0.5712742523639462,
    "p95_s_raw": 0.5712742523639462
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API: no vendor code ran locally. Serving and training code are proprietary (customer statement); the endpoint is TypeSafe-compatible and was driven by the unchanged TypeSafe adapter. Model, customer statement: the merged LoRA adapter launchables/myles-g-31b@0093c518 (gated on Hugging Face) on google/gemma-4-31B-it@842da379. Neither the deployed weights nor the serving stack were independently inspected. TRUNCATION ON THE 15 LONGEST ITEMS (scored as answered): the 15 longest items of this draw (about 77,000 to 82,000 tokens on other tokenizers) were answered, not refused. The endpoint reported only 12,456 to 15,166 input tokens for them, against at most 1,774 for every other item, so it silently truncated their input to its documented 16,384-token context; it returned no refusal and no truncation flag. They are scored as the answers the endpoint gave, as for krino-2b, metask_jev_rain_31b_trained and reflex-instinct-0.6b, which also answered these items on shortened input; the scored-wrong rule applies only to refusals. The cost of the truncation shows up in accuracy, measured: on those 15 items 4 of 15 answers are correct (26.7 %) against 1,234 of 1,485 (83.1 %) on the other 1,485 items (Choice 3 of 10 vs 637 of 740; Noul 0 of 2 vs 335 of 373; Score 1 of 3 vs 262 of 372; a Score item counts as correct within half a label step). MODEL, customer statement: the merged LoRA adapter launchables/myles-g-31b@0093c518 (gated on Hugging Face) on google/gemma-4-31B-it@842da379; serving and training code are proprietary and the API is TypeSafe-compatible. TRAINING AND SELECTION, customer statement: the 231 public items entered model selection (not training); the public set is therefore treated as seen during selection and the normal public-vs-sealed gap rule applies to this row. Customer statements about the model are not independently verified.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 88.81777977591078,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 91.45333932773237
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 88.81777977591078
      },
      "calibration_gen": {
       "accuracy": 0.8217821782178217,
       "cc": 69.49843983156292,
       "cc_by_type": {
        "choice": 84.09331919406152,
        "noul": 52,
        "score": 72.40200030062722
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 69.49843983156292
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 84.38274189852417,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 85.18518518518519,
        "score": 74.4671055510377
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 84.38274189852417
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.98441015141567,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.95323045424703
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.98441015141567
      },
      "jaggedness": {
       "accuracy": 0.6722222222222223,
       "cc": 54.01241281618022,
       "cc_by_type": {
        "choice": 62.463907603464875,
        "noul": 67.56756756756756,
        "score": 32.005763277508215
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 54.01241281618022
      },
      "llm_router": {
       "accuracy": 0.939297124600639,
       "cc": 85.62782046706222,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 95.8762886597938,
        "score": 66.55441232931219
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 85.62782046706222
      },
      "long_state": {
       "accuracy": 0.26666666666666666,
       "cc": -41.32275847934246,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": -100,
        "score": -33.645694792866095
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 65.95778331068276,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 100,
        "score": 47.357886014522464
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 65.95778331068276
      },
      "multilingual": {
       "accuracy": 0.76,
       "cc": 66.0209651405287,
       "cc_by_type": {
        "choice": 70.6553635474405,
        "noul": 69.44444444444444,
        "score": 57.963087429701154
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 66.0209651405287
      },
      "negation_pair_gen": {
       "accuracy": 0.8488372093023255,
       "cc": 69.28604230653808,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 60.00000000000001,
        "score": 54.52479358628089
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 69.28604230653808
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 85.15390034237629,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 100,
        "score": 73.47971904514688
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.15390034237629
      },
      "usecase": {
       "accuracy": 0.7772727272727272,
       "cc": 70.61828397537191,
       "cc_by_type": {
        "choice": 77.94117647058823,
        "noul": 79.16666666666667,
        "score": 54.74700878886081
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 70.61828397537191
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 37.55745153351111,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100,
        "score": -30.184788256609508
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 37.55745153351111
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 75.70280420891712,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 47.84573981569144
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 75.70280420891712
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 88.6005763604217,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 81.73093262108807
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 88.6005763604217
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.60445955038642,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.99959839045162
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.60445955038642
      },
      "en": {
       "accuracy": 0.8392701998262381,
       "cc": 74.81183747161053,
       "cc_by_type": {
        "choice": 83.52473120904064,
        "noul": 79.13669064748201,
        "score": 61.77409055830891
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 74.81183747161053
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 70.84890494528481,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 32.122571279573876
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 70.84890494528481
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 88.8458841272402,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": 99.32453762762225
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 88.8458841272402
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 80.55022576235007,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 84.00361846352078
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 80.55022576235007
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 75.82065942079434,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100,
        "score": 44.221754798695876
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 75.82065942079434
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 58.74455539879628,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -9.736715720992283
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 58.74455539879628
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 40.29914618031202,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 60.00000000000001,
        "score": -10.508439139762915
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 40.29914618031202
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 80.34273879226296,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100,
        "score": 85.47266082123335
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 80.34273879226296
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.77706128881198,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.99785053310259
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.77706128881198
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 39.98237054931828,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 38.56183026267348
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 39.98237054931828
      },
      "no": {
       "accuracy": 0.5,
       "cc": 10.483870967741936,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 10.483870967741936
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 67.66762593881772,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 60.00000000000001,
        "score": 58.237748743241156
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 67.66762593881772
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.6267366064763,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 43.7010701612085
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.6267366064763
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 50.842845370067124,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 43.59236589743543
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 50.842845370067124
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 80.7096626100372,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 63.86811826489422
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 80.7096626100372
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 70.22321689159186,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 42.927715190904635
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 70.22321689159186
      },
      "zh": {
       "accuracy": 1,
       "cc": 85.7067013690126,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 57.120104107037825
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 85.7067013690126
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8533333333333334,
       "cc": 81.0558946784286,
       "cc_by_type": {
        "choice": 81.0558946784286
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.0558946784286
      },
      "noul": {
       "accuracy": 0.8933333333333333,
       "cc": 78.66666666666666,
       "cc_by_type": {
        "noul": 78.66666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 78.66666666666666
      },
      "score": {
       "accuracy": 0.7013333333333334,
       "cc": 59.69977832737404,
       "cc_by_type": {
        "score": 59.69977832737404
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 59.69977832737404
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7727272727272727,
       "cc": 66.27948690725826,
       "cc_by_type": {
        "choice": 84,
        "noul": 75,
        "score": 39.83846072177478
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 66.27948690725826
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 67.70707644780173,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 21.754769716076005
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 67.70707644780173
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 30.911133285722144,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100,
        "score": -58.48611233795551
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 30.911133285722144
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 23.698433433647455,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 100,
        "score": -47.03335466981786
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 23.698433433647455
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 54.16742916528917,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 72.64721503209941
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 54.16742916528917
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": 33.620491366367595,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 66.66666666666667,
        "score": -28.06934351096011
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 33.620491366367595
      },
      "financial_crime": {
       "accuracy": 0.7,
       "cc": 62.696753361107994,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 71.42857142857142,
        "score": 36.26953179200747
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 62.696753361107994
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 62.401951415293944,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 50,
        "score": 68.56473926330344
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 62.401951415293944
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 45.037407616672056,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 100,
        "score": -2.3877771499838385
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 45.037407616672056
      },
      "knowledge_graphs": {
       "accuracy": 0.7727272727272727,
       "cc": 77.07005858523151,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100,
        "score": 56.90610723321059
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 77.07005858523151
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 66.1587119553411,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 32.32626506498971
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 66.1587119553411
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 76.64975776840441,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 94.46540233747132
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 76.64975776840441
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 77.70507938454888,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 73.11523815364664
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 77.70507938454888
      },
      "model_routing": {
       "accuracy": 0.9308755760368663,
       "cc": 83.66920067248388,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100,
        "score": 56.85175786160749
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 83.66920067248388
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 46.62074774073085,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 62.89487278073378
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 46.62074774073085
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 60.69597414311201,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 82.18077294001385
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 60.69597414311201
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 38.31899566170807,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50,
        "score": -5.486362768570374
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 38.31899566170807
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 48.074426494813274,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 68.8609606438601
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 48.074426494813274
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.99391370850913,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.98174112552738
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.99391370850913
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.117788381695,
      86.00359922013621
     ],
     "I": [
      68.58224521990246,
      77.30034626422231
     ],
     "capability": [
      75.03476101966254,
      81.19655150762269
     ],
     "composite": {
      "A": [
       68.95161343706428,
       71.55692630209987
      ],
      "B": [
       68.92716284510078,
       72.62891518757193
      ],
      "C": [
       68.95161343706428,
       71.55692630209987
      ]
     },
     "seed": 44
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.99531596546265,
     "capability": 78.81859215262476,
     "intelligence": 73.64186833978687
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8533333333333334,
      "calibration": 87.13910159795581,
      "cc": 81.0558946784286,
      "cc_by_type": {
       "choice": 81.0558946784286
      },
      "cc_tiered": 80.23536413535999,
      "cc_tiered_open": 79.98663238745361,
      "cc_tiered_sealed": 80.48409588326635,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.0558946784286
     },
     "noul": {
      "accuracy": 0.8933333333333333,
      "calibration": 87.36339653087866,
      "cc": 78.66666666666666,
      "cc_by_type": {
       "noul": 78.66666666666666
      },
      "cc_tiered": 81.49456304620887,
      "cc_tiered_open": 87.52173913043478,
      "cc_tiered_sealed": 75.46738696198295,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 78.66666666666666
     },
     "score": {
      "accuracy": 0.7013333333333334,
      "calibration": 77.48344976755348,
      "cc": 59.69977832737404,
      "cc_by_type": {
       "score": 59.69977832737404
      },
      "cc_tiered": 59.19567783779177,
      "cc_tiered_open": 60.72822271363029,
      "cc_tiered_sealed": 57.66313296195325,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 59.69977832737404
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.99550249391719,
    "Emphasis on Accuracy 60:20:20": 69.50469257996821,
    "Emphasis on Cost 20:20:60": 59.58823989428031,
    "Emphasis on Speed 20:60:20": 73.47661299008504,
    "Intelligence only": 73.64186833978687
   }
  },
  {
   "adapter_id": "images/laya/spec.json:warm_adapter.WarmLoaded",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Convai Innovations",
   "axes": {
    "calibration": 63.22326564159259,
    "cost": 84.89402784895724,
    "intelligence": 1.7910341507793501,
    "speed": 72.744382846694
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.21969249999999993,
      "ece_score": 56.06150000000001,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 56.06150000000001,
      "score_before_time_limit": 56.06150000000001,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.808,
      "brier": 0.28256342088000014,
      "ece": 0.18899546666666664,
      "n": 375,
      "score": 62.20090666666667,
      "score_before_time_limit": 62.20090666666667,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8826666666666667,
      "interval90_width": 0.7128888888888881,
      "n": 375,
      "nrps": 0.18963512817111114,
      "score": 71.4073902581111,
      "score_before_time_limit": 71.4073902581111,
      "time_limit_failed_share": 0,
      "top_ece": 0.19110853333333327
     }
    },
    "score": 63.22326564159259
   },
   "capability": 32.50714989618597,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.0002021673844202101,
     0.06710398502970086
    ],
    "B": [
     0.00012765993788192327,
     0.04494779653575676
    ],
    "C": [
     0.00014039401695847923,
     0.04659998960395892
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0031881034482758625
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Laya (Convai Innovations, ModernBERT-large 421M)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 1.8026531464810367,
    "I_sealed": 1.7794151550776638,
    "base": 1.7910341507793501,
    "excess": -2.5501496510348707,
    "gap": 0.023237991403372904,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 5.40795943944311,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 14.37837837837838,
       "hard": -3.8062283737024116,
       "judge": 16.941331575477918,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -91.30434782608697,
       "judge": -80,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 14.020606060606056,
       "hard": -11.245833333333334,
       "judge": -16.895084745762734,
       "standard": -50.04185185185186
      }
     },
     "sealed|choice": {
      "cc": 5.338245465232991,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 1.5593821594856503,
       "hard": -3.846153846153868,
       "judge": 9.683426443202995,
       "standard": 19.078704273925375
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -56.00000000000001,
       "hard": -84.46601941747574,
       "judge": -76.31578947368422,
       "standard": -74.64788732394366
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 4.452695035460996,
       "hard": -11.828823529411746,
       "judge": -8.140898203592851,
       "standard": -0.2669035532995023
      }
     }
    }
   },
   "jevbench_score": 0.008558710204581235,
   "key": "laya",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6111111111111112,
    "decisive_rate": 0.192,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 63.22326564159259,
     "cost": 84.89402784895724,
     "intelligence": 1.7910341507793501,
     "speed": 73.26302890816496
    },
    "board_release": "v1.6.43",
    "capability": 32.50714989618597,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.00020217712264809318,
      0.06712460770140848
     ],
     "B": [
      0.00012766304420524027,
      0.044955197581294834
     ],
     "C": [
      0.00014040077961673136,
      0.046614310903755875
     ]
    },
    "jevbench_score": 0.008560099345041717,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 189,
    "rank": 233,
    "ranks": {
     "A": 233,
     "B": 233,
     "C": 233,
     "capability": 223
    },
    "scores": {
     "A": 0.008560099345041717,
     "B": 0.005540634651880784,
     "C": 0.005944513434056748
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.800616904720664,
     "p50_s_raw": 0.825308452360332,
     "p95_s_adjusted": 2.619846137147397,
     "p95_s_raw": 1.2349230685736985
    },
    "views": {
     "Balanced 33:33:33": 0.006594024627233607,
     "Emphasis on Accuracy 60:20:20": 0.0037729124634726325,
     "Emphasis on Cost 20:20:60": 0.010563725142022983,
     "Emphasis on Speed 20:60:20": 0.010499068205592536,
     "Intelligence only": 0.0022981141246738437
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 274,
   "ranked": true,
   "ranks": {
    "A": 274,
    "B": 274,
    "C": 274,
    "capability": 264
   },
   "repo": "https://huggingface.co/convaiinnovations/laya",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.008558710204581235,
    "B": 0.0055401690319621935,
    "C": 0.005943548753181413
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 2.0147334575653075,
    "p50_s_raw": 0.9323667287826538,
    "p95_s_adjusted": 2.638421132136136,
    "p95_s_raw": 1.244210566068068
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "ModernBERT-large encoder + option-marker decision head, 421M, RLCD-trained",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      57.988437430874065,
      67.35373438390093
     ],
     "I": [
      0.5052598551452469,
      3.643432797894196
     ],
     "capability": [
      29.732944854873807,
      34.91391985018961
     ],
     "composite": {
      "A": [
       0.0002021673844202101,
       0.06710398502970086
      ],
      "B": [
       0.00012765993788192327,
       0.04494779653575676
      ],
      "C": [
       0.00014039401695847923,
       0.04659998960395892
      ]
     },
     "seed": 56
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 63.22326564159259,
     "capability": 32.50714989618597,
     "intelligence": 1.7910341507793501
    },
    "per_type": {
     "choice": {
      "accuracy": 0.25466666666666665,
      "calibration": 56.06150000000001,
      "cc": 4.682511864952642,
      "cc_by_type": {
       "choice": 4.682511864952642
      },
      "cc_tiered": 5.373102452338051,
      "cc_tiered_open": 5.40795943944311,
      "cc_tiered_sealed": 5.338245465232991,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 4.682511864952642
     },
     "noul": {
      "accuracy": 0.11733333333333333,
      "calibration": 62.20090666666667,
      "cc": -76.53333333333333,
      "cc_by_type": {
       "noul": -76.53333333333333
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2826666666666667,
      "calibration": 71.4073902581111,
      "cc": -7.6405701754385325,
      "cc_by_type": {
       "score": -7.6405701754385325
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 62.99195454673789,
     "capability": 33.098085372560654,
     "intelligence": 3.20421619838342
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.006592925554313149,
    "Emphasis on Accuracy 60:20:20": 0.0037726965510524266,
    "Emphasis on Cost 20:20:60": 0.010562032700695017,
    "Emphasis on Speed 20:60:20": 0.010494054436303035,
    "Intelligence only": 0.0022981141246738437
   }
  },
  {
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Convai Innovations",
   "axes": {
    "calibration": 33.655846666999985,
    "cost": 82.1551651086044,
    "intelligence": 0.2879184668737643,
    "speed": 76.95139513446136
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.44879250000000004,
      "ece_score": 10.241499999999991,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 10.241499999999991,
      "score_before_time_limit": 10.241499999999991,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.28,
      "brier": 0.4157348964266666,
      "ece": 0.3921216000000001,
      "n": 375,
      "score": 21.575679999999974,
      "score_before_time_limit": 21.575679999999974,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.896,
      "interval90_width": 0.7111111111111107,
      "n": 375,
      "nrps": 0.18850373331333325,
      "score": 69.150360001,
      "score_before_time_limit": 69.150360001,
      "time_limit_failed_share": 0,
      "top_ece": 0.21424453333333335
     }
    },
    "score": 33.655846666999985
   },
   "capability": 16.971882566936873,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     0,
     0.016095436642730325
    ],
    "B": [
     0,
     0.01067226036496501
    ],
    "C": [
     0,
     0.011177386557451619
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate; no exact base-model floor applies. $0.01/M input, $0/M output; same-class hosted-encoder estimate; no exact-base market floor.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003933940886699508
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Laya multilingual",
   "endpoint_condition": "evaluator-owned CPU, Sandy (AMD Ryzen 5 3600), 4 of 12 threads, nice -n 5, shared host, offline (HF_HUB_OFFLINE=1)",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 0.3687761432977424,
    "I_sealed": 0.20706079044978623,
    "base": 0.2879184668737643,
    "excess": -2.4116722895902876,
    "gap": 0.16171535284795616,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 1.1063284298932272,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 2.702702702702708,
       "hard": -13.243158225857174,
       "judge": 9.03098220171391,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.28571428571429,
       "hard": -30.434782608695656,
       "judge": -50,
       "standard": -44.44444444444444
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 13.373939393939372,
       "hard": 2.3077777777777753,
       "judge": -8.492881355932202,
       "standard": -37.68148148148147
      }
     },
     "sealed|choice": {
      "cc": 0.6211823713493587,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 2.889660778952063,
       "hard": -4.86425339366518,
       "judge": 5.959031657355701,
       "standard": 2.451040768567572
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 0,
       "hard": -30.097087378640776,
       "judge": -36.8421052631579,
       "standard": -29.5774647887324
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -0.387943262411361,
       "hard": -0.8122549019607739,
       "judge": -1.6932335329341575,
       "standard": -3.1987817258883444
      }
     }
    }
   },
   "jevbench_score": 3.759412198193081e-05,
   "key": "laya-multilingual",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.4962962962962963,
    "decisive_rate": 0.72,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 33.655846666999985,
     "cost": 82.1551651086044,
     "intelligence": 0.2879184668737643,
     "speed": 77.82044954853706
    },
    "board_release": "v1.6.43",
    "capability": 16.971882566936873,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 136,
    "capability_rank": 166,
    "composite_ci95": {
     "A": [
      0,
      0.016100086645554724
     ],
     "B": [
      0,
      0.010673895641907892
     ],
     "C": [
      0,
      0.01118061572607967
     ]
    },
    "jevbench_score": 3.7595668430455566e-05,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 191,
    "rank": 235,
    "ranks": {
     "A": 235,
     "B": 235,
     "C": 235,
     "capability": 238
    },
    "scores": {
     "A": 3.7595668430455566e-05,
     "B": 2.368099374667976e-05,
     "C": 2.6108103076705254e-05
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.8654558090260253,
     "p50_s_raw": 0.3577279045130126,
     "p95_s_adjusted": 1.908579030103283,
     "p95_s_raw": 0.8792895150516415
    },
    "views": {
     "Balanced 33:33:33": 2.843624303166903e-05,
     "Emphasis on Accuracy 60:20:20": 1.5873606797432713e-05,
     "Emphasis on Cost 20:20:60": 4.7066204506481234e-05,
     "Emphasis on Speed 20:60:20": 4.704809343432325e-05,
     "Intelligence only": 9.547035876864281e-06
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 276,
   "ranked": true,
   "ranks": {
    "A": 276,
    "B": 276,
    "C": 276,
    "capability": 279
   },
   "repo": "https://huggingface.co/convaiinnovations/laya/tree/1c5edc17a7acd8701df6fc341c0d179f1c62c982/multilingual",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.759412198193081e-05,
    "B": 2.368050288566296e-05,
    "C": 2.610702915411862e-05
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.953681958001107,
    "p50_s_raw": 0.4018409790005535,
    "p95_s_adjusted": 2.1157138055132236,
    "p95_s_raw": 0.9828569027566117
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "convaiinnovations/laya @ 1c5edc17a7acd8701df6fc341c0d179f1c62c982, multilingual/; mmBERT-base 322M, max_len 1024. Distinct from the published English root checkpoint. Fresh run from item 1 on 29 Sep; interrupted earlier runs were not stitched. In-process, offline, shared AMD Ryzen 5 3600 CPU, four threads. Measured latency includes shared-host conditions; it is not a controlled GPU comparison. The frozen pricing review treats mmBERT as unlisted in the frozen pricing snapshots; the labelled same-class encoder estimate has zero cost delta against BAAI/bge-m3.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      28.738769095461095,
      38.12861857012684
     ],
     "I": [
      0,
      2.2482651295080083
     ],
     "capability": [
      14.566920276697426,
      19.746046606060705
     ],
     "composite": {
      "A": [
       0,
       0.016095436642730325
      ],
      "B": [
       0,
       0.01067226036496501
      ],
      "C": [
       0,
       0.011177386557451619
      ]
     },
     "seed": 57
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 33.655846666999985,
     "capability": 16.971882566936873,
     "intelligence": 0.2879184668737643
    },
    "per_type": {
     "choice": {
      "accuracy": 0.21733333333333332,
      "calibration": 10.241499999999991,
      "cc": -0.09188825630196516,
      "cc_by_type": {
       "choice": -0.09188825630196516
      },
      "cc_tiered": 0.863755400621293,
      "cc_tiered_open": 1.1063284298932272,
      "cc_tiered_sealed": 0.6211823713493587,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0
     },
     "noul": {
      "accuracy": 0.35733333333333334,
      "calibration": 21.575679999999974,
      "cc": -28.53333333333333,
      "cc_by_type": {
       "noul": -28.53333333333333
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.296,
      "calibration": 69.150360001,
      "cc": -1.8657280701754209,
      "cc_by_type": {
       "score": -1.8657280701754209
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 32.7172206036182,
     "capability": 17.460072535779506,
     "intelligence": 2.202924467940814
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.8435063409660868e-05,
    "Emphasis on Accuracy 60:20:20": 1.587338624446757e-05,
    "Emphasis on Cost 20:20:60": 4.706426554955254e-05,
    "Emphasis on Speed 20:60:20": 4.7042281517922134e-05,
    "Intelligence only": 9.547035876864281e-06
   }
  },
  {
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Convai Innovations",
   "axes": {
    "calibration": 82.96996460403702,
    "cost": 82.52942826196218,
    "intelligence": 5.471270906587037,
    "speed": 69.34079458737523
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08964031250000004,
      "ece_score": 82.07193749999999,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 82.07193749999999,
      "score_before_time_limit": 82.07193749999999,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.9946666666666667,
      "brier": 0.24736279077333312,
      "ece": 0.09575093333333361,
      "n": 375,
      "score": 80.84981333333327,
      "score_before_time_limit": 80.84981333333327,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.968,
      "interval90_width": 0.8166666666666665,
      "n": 375,
      "nrps": 0.15890220709111097,
      "score": 85.9881429787778,
      "score_before_time_limit": 85.9881429787778,
      "time_limit_failed_share": 0,
      "top_ece": 0.060667466666666586
     }
    },
    "score": 82.96996460403702
   },
   "capability": 44.22061775531203,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     0.038429115573591126,
     0.5887972663831158
    ],
    "B": [
     0.02532188423731967,
     0.41619886251400867
    ],
    "C": [
     0.02668688581499384,
     0.4088869905438304
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: documented hosted-model estimate; no exact base-model floor applies. $0.01/M input, $0/M output; same-class hosted-encoder estimate; no exact-base market floor.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003822543103448276
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Laya typed-decisions",
   "endpoint_condition": "evaluator-owned CPU, Sandy (AMD Ryzen 5 3600), 4 of 12 threads, nice -n 5, shared host, offline (HF_HUB_OFFLINE=1)",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 5.6928563402143615,
    "I_sealed": 5.249685472959713,
    "base": 5.471270906587037,
    "excess": -2.130216775183595,
    "gap": 0.4431708672546488,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 10.771453572375108,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": -1.9188424032714584,
       "judge": 24.851680949241928,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "open|score": {
      "cc": 6.307115448267977,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 5.935454545454554,
       "hard": -3.607592592592601,
       "judge": 9.858813559322,
       "standard": 20.99481481481481
      }
     },
     "sealed|choice": {
      "cc": 7.102994687643811,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 5.550218017884884,
       "hard": 2.771493212669664,
       "judge": 6.890130353817524,
       "standard": 16.861682473210998
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -96,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "sealed|score": {
      "cc": 8.646061731235328,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 9.860354609929068,
       "hard": 2.728602941176461,
       "judge": 13.044251497005966,
       "standard": 13.276548223350238
      }
     }
    }
   },
   "jevbench_score": 0.21636634154218096,
   "key": "laya-typed-decisions",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 1,
    "decisive_rate": 0.005333333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.96996460403702,
     "cost": 82.52942826196218,
     "intelligence": 5.471270906587037,
     "speed": 70.81965680870755
    },
    "board_release": "v1.6.43",
    "capability": 44.22061775531203,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.03846019800643322,
      0.5898651799347397
     ],
     "B": [
      0.025332676441949146,
      0.41662539701360535
     ],
     "C": [
      0.02670847083780085,
      0.40962859717690253
     ]
    },
    "jevbench_score": 0.21666109526429145,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 170,
    "rank": 214,
    "ranks": {
     "A": 214,
     "B": 214,
     "C": 214,
     "capability": 190
    },
    "scores": {
     "A": 0.21666109526429145,
     "B": 0.14825243435052415,
     "C": 0.1504590939335357
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.8777340129803632,
     "p50_s_raw": 0.8638670064901817,
     "p95_s_adjusted": 4.409610662701016,
     "p95_s_raw": 2.1298053313505076
    },
    "views": {
     "Balanced 33:33:33": 0.17186613033097944,
     "Emphasis on Accuracy 60:20:20": 0.10420153536931118,
     "Emphasis on Cost 20:20:60": 0.2566823909067557,
     "Emphasis on Speed 20:60:20": 0.2523472425383811,
     "Intelligence only": 0.06551257180569732
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 254,
   "ranked": true,
   "ranks": {
    "A": 254,
    "B": 254,
    "C": 254,
    "capability": 229
   },
   "repo": "https://huggingface.co/convaiinnovations/laya/tree/1c5edc17a7acd8701df6fc341c0d179f1c62c982/typed-decisions",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.21636634154218096,
    "B": 0.1481419607342047,
    "C": 0.15025440384873676
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 2.1531285819859476,
    "p50_s_raw": 1.0015642909929738,
    "p95_s_adjusted": 5.405683011900692,
    "p95_s_raw": 2.6278415059503457
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "convaiinnovations/laya @ 1c5edc17a7acd8701df6fc341c0d179f1c62c982, typed-decisions/; ModernBERT-large, max_len 1024, distinct head and calibration from the English root. Fresh full run on 29 Sep; in-process, offline, shared AMD Ryzen 5 3600 CPU, four threads. Measured latency includes shared-host conditions; it is not a controlled GPU comparison. Complete valid probabilities on all types; frequent noul abstention lowers Intelligence to its zero floor and thus the composite to zero. This is not a transport failure.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.7245419121787,
      86.71414939090002
     ],
     "I": [
      2.9916515598721545,
      7.823617519507115
     ],
     "capability": [
      41.079834277143355,
      46.819056265617796
     ],
     "composite": {
      "A": [
       0.038429115573591126,
       0.5887972663831158
      ],
      "B": [
       0.02532188423731967,
       0.41619886251400867
      ],
      "C": [
       0.02668688581499384,
       0.4088869905438304
      ]
     },
     "seed": 58
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.96996460403702,
     "capability": 44.22061775531203,
     "intelligence": 5.471270906587037
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2733333333333333,
      "calibration": 82.07193749999999,
      "cc": 7.069711925579947,
      "cc_by_type": {
       "choice": 7.069711925579947
      },
      "cc_tiered": 8.93722413000946,
      "cc_tiered_open": 10.771453572375108,
      "cc_tiered_sealed": 7.102994687643811,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 7.069711925579947
     },
     "noul": {
      "accuracy": 0.005333333333333333,
      "calibration": 80.84981333333327,
      "cc": -98.93333333333332,
      "cc_by_type": {
       "noul": -98.93333333333332
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.36533333333333334,
      "calibration": 85.9881429787778,
      "cc": 7.23879824561402,
      "cc_by_type": {
       "score": 7.23879824561402
      },
      "cc_tiered": 7.476588589751652,
      "cc_tiered_open": 6.307115448267977,
      "cc_tiered_sealed": 8.646061731235328,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 7.23879824561402
     }
    },
    "raw_on_A": {
     "calibration": 79.92527034876068,
     "capability": 42.6053892600956,
     "intelligence": 5.28550817143052
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.17161885391660758,
    "Emphasis on Accuracy 60:20:20": 0.10414694710182727,
    "Emphasis on Cost 20:20:60": 0.25635140429607584,
    "Emphasis on Speed 20:60:20": 0.25138994625628774,
    "Intelligence only": 0.06551257180569732
   }
  },
  {
   "adapter_id": "jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Interfaze",
   "axes": {
    "calibration": 82.32371666823741,
    "cost": 76.34084325677026,
    "intelligence": 41.084702071234844,
    "speed": 87.52560274254004
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09976787636606844,
      "ece_score": 80.04642472678631,
      "mean_tvd": null,
      "n": 735,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.04642472678631,
      "score_before_time_limit": 80.04642472678631,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.15633423180592987,
      "brier": 0.15547862054178035,
      "ece": 0.09804350768859321,
      "n": 371,
      "score": 80.39129846228136,
      "score_before_time_limit": 80.39129846228136,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9728997289972899,
      "interval90_width": 0.7095754290876237,
      "n": 369,
      "nrps": 0.12033260727134382,
      "score": 86.53342681564456,
      "score_before_time_limit": 86.53342681564456,
      "time_limit_failed_share": 0,
      "top_ece": 0.07449942820788243
     }
    },
    "score": 82.32371666823741
   },
   "capability": 61.70420936973613,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,015,861 input + 0 output tokens over 1,624 decisions = 625.530172414 tokens/decision. GPU-time cost USD 0.00614665104308104/1,000; old list-price cost USD 0.0187659051724138/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00614665104308104/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.006146651043081042
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "lev (Interfaze AI, Qwen3.5-4B + LoRA r32, label-token readout + candidate-path head, per-bucket temperatures)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline (HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE), credential-free, server bound to 127.0.0.1, harness on the same pod over loopback HTTP; Lium pod 08a1ff4a-248a-4ff2-948e-ea558f99e808 (huid cosmic-hawk-4f, node calm-fox-65), 1x NVIDIA H100 80GB HBM3, United States, USD 1.30/h",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 45.778585099687206,
    "I_sealed": 36.39081904278248,
    "base": 41.084702071234844,
    "excess": 6.81437841446648,
    "gap": 9.387766056904724,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 61.009098246344166,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 50.92796476879523,
       "judge": 68.35860250494396,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 56.946169772256724,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 73.91304347826086,
       "judge": 50,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 19.38048728046074,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 22.564349870758527,
       "hard": 19.589685248514364,
       "judge": 26.558897550056017,
       "standard": 6.602544644811692
      }
     },
     "sealed|choice": {
      "cc": 54.52900312820748,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 78.71554208853745,
       "hard": 48.585972850678715,
       "judge": 54.37616387337058,
       "standard": 54.55105308535534
      }
     },
     "sealed|noul": {
      "cc": 29.603595615594433,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 63.99999999999999,
       "hard": 2.9126213592232997,
       "judge": 36.8421052631579,
       "standard": 54.92957746478873
      }
     },
     "sealed|score": {
      "cc": 25.039858384545557,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 47.25762460028372,
       "hard": 16.968805028678304,
       "judge": 35.272820946831395,
       "standard": 14.723638144982221
      }
     }
    }
   },
   "jevbench_score": 44.265822247960145,
   "key": "lev",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "interfaze-ai/lev@7bdc748dffebd85b57ee0dbea8f994c6354fed31 + Abhinavexists/lev@cf104b69329302e4eac674a730c71f3511047db8",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8274760383386581,
    "decisive_rate": 0.8436657681940701,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.32371666823741,
     "cost": 61.79891447724963,
     "intelligence": 41.084702071234844,
     "speed": 88.00251499520694
    },
    "board_release": "v1.6.43",
    "capability": 61.70420936973613,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 55,
    "capability_rank": 80,
    "composite_ci95": {
     "A": [
      28.718912479130207,
      50.73774516604421
     ],
     "B": [
      25.322064520088336,
      46.59426967064348
     ],
     "C": [
      19.943689221618207,
      35.23454525419738
     ]
    },
    "jevbench_score": 42.177760514319104,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 50,
    "rank": 77,
    "ranks": {
     "A": 77,
     "B": 80,
     "C": 81,
     "capability": 124
    },
    "scores": {
     "A": 42.177760514319104,
     "B": 38.201113541708786,
     "C": 29.290111468277157
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3546039809356444,
     "p50_s_raw": 0.10230199046782218,
     "p95_s_adjusted": 0.446688625391107,
     "p95_s_raw": 0.1483443126955535
    },
    "views": {
     "Balanced 33:33:33": 39.03925349464146,
     "Emphasis on Accuracy 60:20:20": 33.569486237063714,
     "Emphasis on Cost 20:20:60": 40.07113476989396,
     "Emphasis on Speed 20:60:20": 45.246505158344306,
     "Intelligence only": 27.739614243661805
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 134,
   "ranked": true,
   "ranks": {
    "A": 134,
    "B": 141,
    "C": 139,
    "capability": 155
   },
   "repo": "https://huggingface.co/interfaze-ai/lev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 44.265822247960145,
    "B": 39.55298872980945,
    "C": 30.740154338861213
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.36095349994720893,
    "p50_s_raw": 0.10547674997360446,
    "p95_s_adjusted": 0.489765814790735,
    "p95_s_raw": 0.16988290739536752
   },
   "status": {
    "answered_ok": 1475,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.63846170548479,
      84.6851724512148
     ],
     "I": [
      35.04763127374306,
      44.44652580243387
     ],
     "capability": [
      56.747890795417284,
      63.66051033763406
     ],
     "composite": null,
     "seed": 59
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.32371666823741,
     "capability": 61.70420936973613,
     "intelligence": 41.084702071234844
    },
    "per_type": {
     "choice": {
      "accuracy": 0.664,
      "calibration": 80.04642472678631,
      "cc": 57.030398908708534,
      "cc_by_type": {
       "choice": 57.030398908708534
      },
      "cc_tiered": 57.76905068727582,
      "cc_tiered_open": 61.009098246344166,
      "cc_tiered_sealed": 54.52900312820748,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.030398908708534
     },
     "noul": {
      "accuracy": 0.6906666666666667,
      "calibration": 80.39129846228136,
      "cc": 38.13333333333333,
      "cc_by_type": {
       "noul": 38.13333333333333
      },
      "cc_tiered": 43.274882693925576,
      "cc_tiered_open": 56.946169772256724,
      "cc_tiered_sealed": 29.603595615594433,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 38.13333333333333
     },
     "score": {
      "accuracy": 0.4533333333333333,
      "calibration": 86.53342681564456,
      "cc": 23.66953381268505,
      "cc_by_type": {
       "score": 23.66953381268505
      },
      "cc_tiered": 22.210172832503147,
      "cc_tiered_open": 19.38048728046074,
      "cc_tiered_sealed": 25.039858384545557,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 23.66953381268505
     }
    },
    "raw_on_A": {
     "calibration": 79.50973335881918,
     "capability": 61.67507354014826,
     "intelligence": 43.84041372147735
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.020000000000000018,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 41.45238533431319,
    "Emphasis on Accuracy 60:20:20": 34.608962420274416,
    "Emphasis on Cost 20:20:60": 44.9745152119786,
    "Emphasis on Speed 20:60:20": 47.074046369446904,
    "Intelligence only": 27.739614243661805
   }
  },
  {
   "adapter_id": "images/lev-350m/spec.json:lev_v15.LevAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Franck Verrot (franckverrot)",
   "axes": {
    "calibration": 81.86424479313416,
    "cost": 80.03797760430007,
    "intelligence": 4.827562367090417,
    "speed": 93.7625770756901
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04337133550488594,
      "ece_score": 91.32573289902281,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 91.32573289902281,
      "score_before_time_limit": 91.32573289902281,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.921832884097035,
      "brier": 0.2643957088948787,
      "ece": 0.12656064690026958,
      "n": 371,
      "score": 74.68787061994608,
      "score_before_time_limit": 74.68787061994608,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.978319783197832,
      "interval90_width": 0.9372177055103887,
      "n": 369,
      "nrps": 0.17965857520325207,
      "score": 79.5791308604336,
      "score_before_time_limit": 79.5791308604336,
      "time_limit_failed_share": 0,
      "top_ece": 0.11437940379403796
     }
    },
    "score": 81.86424479313416
   },
   "capability": 43.34590358011229,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.036594537252326705,
     0.39115195111968926
    ],
    "B": [
     0.024011565429306884,
     0.2702559115870675
    ],
    "C": [
     0.02541287309189354,
     0.2716332993886731
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00462807881773399
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "lev-350m (Franck Verrot, LFM2.5-350M)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 4.690739555547254,
    "I_sealed": 4.964385178633581,
    "base": 4.827562367090417,
    "excess": -2.8470332655245705,
    "gap": -0.2736456230863267,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 14.072218666641763,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486491,
       "hard": 11.292859389745214,
       "judge": 16.941331575477918,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 16.442424242424224,
       "hard": -5.066666666666664,
       "judge": 1.4508474576270802,
       "standard": -32.55555555555556
      }
     },
     "sealed|choice": {
      "cc": 14.893155535900744,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 21.513561451481806,
       "hard": 9.898190045248851,
       "judge": 14.338919925512121,
       "standard": 22.404236974996934
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -76,
       "hard": -92.23300970873787,
       "judge": -81.57894736842105,
       "standard": -94.36619718309859
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 15.455319148936175,
       "hard": -14.996568627450912,
       "judge": -3.196407185628769,
       "standard": -22.131472081218284
      }
     }
    }
   },
   "jevbench_score": 0.1537555644865392,
   "key": "lev-350m",
   "last_measured_on": "2026-10-05",
   "licence": "Apache-2.0 (code); weights under LiquidAI's LFM1.0 licence, following the LFM2.5-350M base",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6896551724137931,
    "decisive_rate": 0.07816711590296496,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.86424479313416,
     "cost": 80.03797760430007,
     "intelligence": 4.827562367090417,
     "speed": 93.98230372494703
    },
    "board_release": "v1.6.43",
    "capability": 43.34590358011229,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 105,
    "capability_rank": 133,
    "composite_ci95": {
     "A": [
      0.036596960876541004,
      0.39120491517125294
     ],
     "B": [
      0.024012400167042817,
      0.27027613731690037
     ],
     "C": [
      0.02541455616426458,
      0.2716700799800367
     ]
    },
    "jevbench_score": 0.15377137469809216,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 174,
    "rank": 218,
    "ranks": {
     "A": 218,
     "B": 219,
     "C": 218,
     "capability": 194
    },
    "scores": {
     "A": 0.15377137469809216,
     "B": 0.1036628826651748,
     "C": 0.10678567687367513
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1946538500036695,
     "p50_s_raw": 0.022326925001834752,
     "p95_s_adjusted": 0.2053556374099571,
     "p95_s_raw": 0.02767781870497856
    },
    "views": {
     "Balanced 33:33:33": 0.12144625080361596,
     "Emphasis on Accuracy 60:20:20": 0.07231333345467754,
     "Emphasis on Cost 20:20:60": 0.18259637420585254,
     "Emphasis on Speed 20:60:20": 0.1852875240477895,
     "Intelligence only": 0.04500322848109108
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 259,
   "ranked": true,
   "ranks": {
    "A": 259,
    "B": 260,
    "C": 259,
    "capability": 233
   },
   "repo": "https://github.com/franckverrot/lev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.1537555644865392,
    "B": 0.10365713430715648,
    "C": 0.1067746975600967
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19497151399846188,
    "p50_s_raw": 0.022485756999230944,
    "p95_s_adjusted": 0.21566075850132618,
    "p95_s_raw": 0.032830379250663094
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "LiquidAI/LFM2.5-350M with a LoRA and a 6.5M-parameter pointer head (a kev clone)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.71542881918619,
      83.56232508492258
     ],
     "I": [
      2.9345502485864183,
      6.710083033760257
     ],
     "capability": [
      40.03125074179408,
      44.53569592988895
     ],
     "composite": {
      "A": [
       0.036594537252326705,
       0.39115195111968926
      ],
      "B": [
       0.024011565429306884,
       0.2702559115870675
      ],
      "C": [
       0.02541287309189354,
       0.2716332993886731
      ]
     },
     "seed": 60
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.86424479313416,
     "capability": 43.34590358011229,
     "intelligence": 4.827562367090417
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3333333333333333,
      "calibration": 91.32573289902281,
      "cc": 14.74285497759628,
      "cc_by_type": {
       "choice": 14.74285497759628
      },
      "cc_tiered": 14.482687101271253,
      "cc_tiered_open": 14.072218666641763,
      "cc_tiered_sealed": 14.893155535900744,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 14.74285497759628
     },
     "noul": {
      "accuracy": 0.05333333333333334,
      "calibration": 74.68787061994608,
      "cc": -89.33333333333333,
      "cc_by_type": {
       "noul": -89.33333333333333
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.248,
      "calibration": 79.5791308604336,
      "cc": -8.448333333333302,
      "cc_by_type": {
       "score": -8.448333333333302
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 81.0219883791457,
     "capability": 43.49027409042074,
     "intelligence": 5.958559801695786
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.12143310184738997,
    "Emphasis on Accuracy 60:20:20": 0.07231053614020283,
    "Emphasis on Cost 20:20:60": 0.1825785396126524,
    "Emphasis on Speed 20:60:20": 0.18523244254058632,
    "Intelligence only": 0.04500322848109108
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Liquid AI",
   "axes": {
    "calibration": 86.78599512136702,
    "cost": 62.84038569943876,
    "intelligence": 62.05738930369758,
    "speed": 87.8529201384932
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07532022247062888,
      "ece_score": 84.93595550587423,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 84.93595550587423,
      "score_before_time_limit": 84.93595550587423,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.17333333333333334,
      "brier": 0.0993520721969318,
      "ece": 0.07450471425946179,
      "n": 375,
      "score": 85.09905714810763,
      "score_before_time_limit": 85.09905714810763,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9386666666666666,
      "interval90_width": 0.28222222222222215,
      "n": 375,
      "nrps": 0.06855989887138964,
      "score": 90.32297271011922,
      "score_before_time_limit": 90.32297271011922,
      "time_limit_failed_share": 0,
      "top_ece": 0.062490323463112944
     }
    },
    "score": 86.78599512136702
   },
   "capability": 74.4216922125323,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     70.39745141854905,
     74.05433680202529
    ],
    "B": [
     67.24590136651118,
     72.27360295159241
    ],
    "C": [
     63.53024774822211,
     74.05433680202529
    ]
   },
   "cost": {
    "basis": "Liquid AI d1 public API tariff USD 0.04 per 1M input tokens, no output tokens (Vercel AI Gateway listing https://vercel.com/ai-gateway/models/d1, read 6 Oct 2026; equals Liquid API usage.cost); measured on the v1.6.1 common cost basis from this run's token usage",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.050374800000000004
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.017324224779959378
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "d1 (Liquid AI)",
   "endpoint_condition": "Liquid AI hosted API, text only, measured from Sandy (Helsinki) on 6 Oct 2026; full set 1,200 sealed + P300 (v1.6.1), not equated.",
   "endpoint_kind": "api",
   "full_api_addon": {
    "measured_on": "2026-10-06",
    "results_sha256": "e8af18a525745185563af558f24c97605cc4c5402d945dc9ce8ad5598ffe2606",
    "round": "a4-3"
   },
   "full_coverage": true,
   "intelligence": {
    "I_open": 64.87457995498468,
    "I_sealed": 59.240198652410484,
    "base": 62.05738930369758,
    "excess": 3.060993660135954,
    "gap": 5.634381302574198,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.4599081792597,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 71.68921044353571,
       "judge": 72.31377719182596,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 61.42028985507247,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 57.74354183062191,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 59.53512949354776,
       "hard": 58.97474523537042,
       "judge": 58.364773735967425,
       "standard": 53.453493331643685
      }
     },
     "sealed|choice": {
      "cc": 71.1719233348168,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 78.71554208853745,
       "hard": 64.87556561085972,
       "judge": 71.13594040968344,
       "standard": 80.04680379357065
      }
     },
     "sealed|noul": {
      "cc": 46.350422823090824,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 22.330097087378633,
       "judge": 55.263157894736835,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 60.19824979932386,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 76.02006250272461,
       "hard": 54.94396966222077,
       "judge": 65.34461858109049,
       "standard": 55.07635054917971
      }
     }
    }
   },
   "jevbench_score": 72.81636405537589,
   "key": "liquid-d1",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary hosted API",
   "listing": "ranked",
   "method": "v1.7",
   "noul_decisive": {
    "acc_among_decisive": 0.9258064516129032,
    "decisive_rate": 0.8266666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 10,
    "axes": {
     "calibration": 86.78599512136702,
     "cost": 62.84038569943876,
     "intelligence": 62.05738930369758,
     "speed": 89.22497122158413
    },
    "board_release": "v1.6.43",
    "capability": 74.4216922125323,
    "capability_api_board_rank": 11,
    "capability_open_board_rank": null,
    "capability_rank": 25,
    "composite_ci95": {
     "A": [
      70.61498257469916,
      74.29509345786346
     ],
     "B": [
      67.40457796119317,
      72.4569260841226
     ],
     "C": [
      63.72660803369401,
      74.29509345786346
     ]
    },
    "jevbench_score": 73.0491258431614,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 13,
    "ranks": {
     "A": 13,
     "B": 19,
     "C": 13,
     "capability": 47
    },
    "scores": {
     "A": 73.0491258431614,
     "B": 70.54993499095869,
     "C": 73.0491258431614
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.26846902817487717,
     "p50_s_raw": 0.26846902817487717,
     "p95_s_adjusted": 0.44525487907230826,
     "p95_s_raw": 0.44525487907230826
    },
    "views": {
     "Balanced 33:33:33": 69.38810749276507,
     "Emphasis on Accuracy 60:20:20": 66.25736619060567,
     "Emphasis on Cost 20:20:60": 66.61183085641215,
     "Emphasis on Speed 20:60:20": 76.16108626295785,
     "Intelligence only": 62.05738930369758
    }
   },
   "rank": 48,
   "ranked": true,
   "ranks": {
    "A": 48,
    "B": 50,
    "C": 39,
    "capability": 65
   },
   "repo": "https://www.liquid.ai/blog/d1-decision-model",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 72.81636405537589,
    "B": 70.37612304785493,
    "C": 72.81636405537589
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26902734860777855,
    "p50_s_raw": 0.26902734860777855,
    "p95_s_adjusted": 0.609412775933742,
    "p95_s_raw": 0.609412775933742
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.98139688401669,
      88.21215531212913
     ],
     "I": [
      56.94190030827255,
      65.96370651107074
     ],
     "capability": [
      70.07219589958059,
      76.5079704884497
     ],
     "composite": {
      "A": [
       70.39745141854905,
       74.05433680202529
      ],
      "B": [
       67.24590136651118,
       72.27360295159241
      ],
      "C": [
       63.53024774822211,
       74.05433680202529
      ]
     },
     "seed": 115
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.78599512136702,
     "capability": 74.4216922125323,
     "intelligence": 62.05738930369758
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7866666666666666,
      "calibration": 84.93595550587423,
      "cc": 72.7177135928308,
      "cc_by_type": {
       "choice": 72.7177135928308
      },
      "cc_tiered": 73.31591575703825,
      "cc_tiered_open": 75.4599081792597,
      "cc_tiered_sealed": 71.1719233348168,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 72.7177135928308
     },
     "noul": {
      "accuracy": 0.7653333333333333,
      "calibration": 85.09905714810763,
      "cc": 53.06666666666666,
      "cc_by_type": {
       "noul": 53.06666666666666
      },
      "cc_tiered": 53.885356339081646,
      "cc_tiered_open": 61.42028985507247,
      "cc_tiered_sealed": 46.350422823090824,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 53.06666666666666
     },
     "score": {
      "accuracy": 0.7066666666666667,
      "calibration": 90.32297271011922,
      "cc": 59.75373554858271,
      "cc_by_type": {
       "score": 59.75373554858271
      },
      "cc_tiered": 58.970895814972884,
      "cc_tiered_open": 57.74354183062191,
      "cc_tiered_sealed": 60.19824979932386,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 59.75373554858271
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.10832417715342,
    "Emphasis on Accuracy 60:20:20": 66.1040387730098,
    "Emphasis on Cost 20:20:60": 66.45686042235283,
    "Emphasis on Speed 20:60:20": 75.5567408098811,
    "Intelligence only": 62.05738930369758
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Liquid AI",
   "axes": {
    "calibration": 76.87411794897184,
    "cost": 50.56108621702079,
    "intelligence": 22.13919485880221,
    "speed": 94.0900145591257
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11313979352292604,
      "ece_score": 77.37204129541479,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 77.37204129541479,
      "score_before_time_limit": 77.37204129541479,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.5093333333333334,
      "brier": 0.19849004201230716,
      "ece": 0.11360395591990471,
      "n": 375,
      "score": 77.27920881601906,
      "score_before_time_limit": 77.27920881601906,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9093333333333333,
      "interval90_width": 0.5644444444444446,
      "n": 375,
      "nrps": 0.1509830793884296,
      "score": 75.9711037354817,
      "score_before_time_limit": 75.9711037354817,
      "time_limit_failed_share": 0,
      "top_ece": 0.16479742295096816
     }
    },
    "score": 76.87411794897184
   },
   "capability": 49.50665640388703,
   "composite_ci95": {
    "A": [
     5.217354515822397,
     12.201151252166042
    ],
    "B": [
     4.192329175683889,
     10.346538617725733
    ],
    "C": [
     3.6231628582099975,
     8.473021702893085
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no bookable listing for this base. The declared base LiquidAI/LFM2.5-VL-3B is not a key of any frozen pricing table, and the frozen 25 Sep 2026 snapshot's only Liquid entry is liquid/lfm-2.5-2.6b:free at (0, 0) - a free route, which may not price a row (free-tier rule: use the published paid price or the estimate). Method, as aplomb-1, wald-4b-v2, jiwo-* and the ClassOne trio carry: the DeepInfra Qwen3.5 size-class reference from the frozen snapshot, rounded ONE CLASS UP so the estimate can only OVERSTATE this row's cost - deepinfra:Qwen/Qwen3.5-4B at USD 0.03 per 1M input and USD 0.15 per 1M output, for a 3B model. Applied to this row's own measured usage: 2,222,971 input tokens and 0 output tokens over 1,500 decisions. The 2B rate (0.02, 0.10) would give 0.029639613333333335 and was deliberately NOT used because it understates. Precedent for a LiquidAI base priced as a labelled estimate: the live lev-350m row (LiquidAI/LFM2.5-350M, in UNLISTED_BASE_MODELS, estimate 0.00462807881773399 per 1,000).",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.044459420000000006
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Liquid AI d1-3B",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX 5090 32 GB pod (France), offline after setup (egress cut and proven), author's custom code at the pinned commit via trust_remote_code from the sha256-verified local copy, bfloat16, no torch.compile",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 27.85582362507926,
    "I_sealed": 16.42256609252516,
    "base": 22.13919485880221,
    "excess": 1.3600129512143884,
    "gap": 11.433257532554101,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 53.72008003683186,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675676,
       "hard": 43.378420887071414,
       "judge": 60.44825313117996,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -30.434782608695656,
       "judge": -19.999999999999996,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 29.847390838405925,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.55757331890299,
       "hard": 17.69560694707587,
       "judge": 41.70520484649972,
       "standard": 33.00914636867683
      }
     },
     "sealed|choice": {
      "cc": 39.056033872257295,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 54.948378350192925,
       "hard": 33.21974277234252,
       "judge": 38.815889604789206,
       "standard": 43.14266023432116
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 7.692307692307687,
       "hard": -45.45454545454546,
       "judge": -5.555555555555558,
       "standard": -5.263157894736848
      }
     },
     "sealed|score": {
      "cc": 10.211664405318189,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 27.92356315993724,
       "hard": 9.442062390485162,
       "judge": 0.13958879484807785,
       "standard": 18.003032473379875
      }
     }
    }
   },
   "jevbench_score": 8.853035001087052,
   "key": "liquid-d1-3b",
   "last_measured_on": "2026-10-10",
   "licence": "LFM Open License v1.0 (commercial use only below USD 10 M annual revenue)",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "LiquidAI/d1-3B@051bcc464b01b9f92942b364d9586b0ef5912432",
   "noul_decisive": {
    "acc_among_decisive": 0.8315217391304348,
    "decisive": 184,
    "decisive_rate": 0.49066666666666664,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 76.87411794897184,
     "cost": 50.56108621702079,
     "intelligence": 22.13919485880221,
     "speed": 94.32773191369529
    },
    "board_release": "v1.6.43",
    "capability": 49.50665640388703,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 91,
    "capability_rank": 118,
    "composite_ci95": {
     "A": [
      5.218755487589073,
      12.205081733672536
     ],
     "B": [
      4.193052691245517,
      10.348794476091657
     ],
     "C": [
      3.6241357552701894,
      8.475751203939263
     ]
    },
    "jevbench_score": 8.855712627780655,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 136,
    "rank": 175,
    "ranks": {
     "A": 175,
     "B": 176,
     "C": 176,
     "capability": 175
    },
    "scores": {
     "A": 8.855712627780655,
     "B": 7.330614913163864,
     "C": 6.149800435958788
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1797857349505648,
     "p50_s_raw": 0.0148928674752824,
     "p95_s_adjusted": 0.20533906437922267,
     "p95_s_raw": 0.027669532189611334
    },
    "views": {
     "Balanced 33:33:33": 7.785401938124825,
     "Emphasis on Accuracy 60:20:20": 5.909418623202946,
     "Emphasis on Cost 20:20:60": 8.516521034074662,
     "Emphasis on Speed 20:60:20": 10.132088187487657,
     "Intelligence only": 4.340556958274228
    }
   },
   "rank": 218,
   "ranked": true,
   "ranks": {
    "A": 218,
    "B": 219,
    "C": 219,
    "capability": 212
   },
   "repo": "https://huggingface.co/LiquidAI/d1-3B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 8.853035001087052,
    "B": 7.329146939870814,
    "C": 6.14794097297712
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.18326355200260877,
    "p50_s_raw": 0.016631776001304388,
    "p95_s_adjusted": 0.21277590398676693,
    "p95_s_raw": 0.03138795199338346
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7241379310344828,
       "cc": 54.390853941466354,
       "cc_by_type": {
        "choice": 75,
        "noul": 66.66666666666667,
        "score": 21.50589515773239
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 54.390853941466354
      },
      "calibration_gen": {
       "accuracy": 0.45535714285714285,
       "cc": 7.885435170467154,
       "cc_by_type": {
        "choice": 41.770186335403736,
        "noul": 0,
        "score": -18.113880824002273
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 7.885435170467154
      },
      "composite_gen": {
       "accuracy": 0.5408163265306123,
       "cc": 25.303042500575433,
       "cc_by_type": {
        "choice": 77.77777777777779,
        "noul": -21.42857142857143,
        "score": 19.559921152519944
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 25.303042500575433
      },
      "fairness": {
       "accuracy": 0.8518518518518519,
       "cc": 47.39465841291355,
       "cc_by_type": {
        "choice": 88.23529411764707,
        "noul": 0,
        "score": 53.948681121093585
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 47.39465841291355
      },
      "jaggedness": {
       "accuracy": 0.31891891891891894,
       "cc": -22.274181763566684,
       "cc_by_type": {
        "choice": 20.79666512274201,
        "noul": -79.48717948717949,
        "score": -8.13203092626258
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.625,
       "cc": 51.3034512100294,
       "cc_by_type": {
        "choice": 47.05062703204832,
        "noul": 50.68493150684932,
        "score": 56.17479509119057
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 51.3034512100294
      },
      "long_state": {
       "accuracy": 0.1724137931034483,
       "cc": -29.57896952664178,
       "cc_by_type": {
        "choice": 16.66666666666667,
        "noul": -100,
        "score": -5.403575246592007
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2903225806451613,
       "cc": -13.862612846690979,
       "cc_by_type": {
        "choice": 7.692307692307688,
        "noul": -50,
        "score": 0.7198537676193717
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.41904761904761906,
       "cc": 4.0623722132052436,
       "cc_by_type": {
        "choice": 34.617107030900144,
        "noul": -33.333333333333336,
        "score": 10.903342942048921
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 4.0623722132052436
      },
      "negation_pair_gen": {
       "accuracy": 0.44642857142857145,
       "cc": -3.4722633613639875,
       "cc_by_type": {
        "choice": 38.095238095238095,
        "noul": -50,
        "score": 1.4879718206699422
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.7017543859649122,
       "cc": 62.264150943396224,
       "cc_by_type": {
        "choice": 62.264150943396224
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 62.264150943396224
      },
      "perturb_pair": {
       "accuracy": 0.7166666666666667,
       "cc": 57.31023460546714,
       "cc_by_type": {
        "choice": 52.81782437745741,
        "noul": 42.85714285714286,
        "score": 76.25573658180112
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 57.31023460546714
      },
      "usecase": {
       "accuracy": 0.39662447257383965,
       "cc": -1.9444617361056398,
       "cc_by_type": {
        "choice": 38.716515706301806,
        "noul": -43.85964912280702,
        "score": -0.6902517918117024
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.18181818181818182,
       "cc": -100.14332542233501,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": -100,
        "score": -220.429976267005
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.375,
       "cc": -4.802259887005647,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.375,
       "cc": -8.98770932887382,
       "cc_by_type": {
        "choice": 17.159763313609464,
        "noul": -60,
        "score": 15.877108699769082
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.4222222222222222,
       "cc": 6.322815228633009,
       "cc_by_type": {
        "choice": 34.8219674109837,
        "noul": -11.111111111111116,
        "score": -4.742410613973558
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 6.322815228633009
      },
      "el": {
       "accuracy": 0.2777777777777778,
       "cc": -39.39660414361751,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -100,
        "score": -33.85246303326217
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.4924039320822163,
       "cc": 14.003345625582787,
       "cc_by_type": {
        "choice": 44.13909141895677,
        "noul": -18.045112781954884,
        "score": 15.916058239746478
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 14.003345625582787
      },
      "es": {
       "accuracy": 0.6428571428571429,
       "cc": 33.758516235110754,
       "cc_by_type": {
        "choice": 73.81818181818181,
        "noul": 27.27272727272727,
        "score": 0.18463961442317656
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 33.758516235110754
      },
      "fi": {
       "accuracy": 0,
       "cc": -63.38028169014085,
       "cc_by_type": {
        "choice": -26.760563380281692,
        "noul": -100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.6,
       "cc": 28.727877312438604,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -50,
        "score": 36.183631937315795
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 28.727877312438604
      },
      "hi": {
       "accuracy": 0.3,
       "cc": 1.3860870885139314,
       "cc_by_type": {
        "choice": 13.043478260869566,
        "noul": -19.999999999999996,
        "score": 11.114783004672224
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 1.3860870885139314
      },
      "id": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 17.741935483870968
      },
      "it": {
       "accuracy": 0.6470588235294118,
       "cc": 40.934769362003244,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 50,
        "score": 23.5800485655584
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 40.934769362003244
      },
      "ja": {
       "accuracy": 0.34375,
       "cc": -7.243137787326186,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": -42.85714285714286,
        "score": 2.428542503294384
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.5454545454545454,
       "cc": 21.42666905349395,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 0,
        "score": 29.853777652285117
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 21.42666905349395
      },
      "mixed": {
       "accuracy": 0.7272727272727273,
       "cc": 49.321740609526785,
       "cc_by_type": {
        "choice": 78.02197802197803,
        "noul": 0,
        "score": 69.94324380660233
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 49.321740609526785
      },
      "nl": {
       "accuracy": 0.38461538461538464,
       "cc": 19.828553848785166,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": 0,
        "score": 59.76421307838892
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 19.828553848785166
      },
      "no": {
       "accuracy": 0.42857142857142855,
       "cc": 17.486338797814206,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 17.486338797814206
      },
      "pl": {
       "accuracy": 0.525,
       "cc": 29.992507104770652,
       "cc_by_type": {
        "choice": 84.25885439440316,
        "noul": 0,
        "score": 5.718666919908799
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 29.992507104770652
      },
      "pt": {
       "accuracy": 0.6666666666666666,
       "cc": 54.789361628198456,
       "cc_by_type": {
        "choice": 63.81909547738693,
        "noul": 33.33333333333333,
        "score": 67.2156560738751
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 54.789361628198456
      },
      "sv": {
       "accuracy": 0,
       "cc": -60.92596627415683,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": -49.44456548913716
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.45454545454545453,
       "cc": -25.14391404022173,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": -33.333333333333336,
        "score": -115.78261931364766
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.5454545454545454,
       "cc": -3.966886884613037,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100,
        "score": 53.3167306505087
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 27.7843293385466,
       "cc_by_type": {
        "choice": 2.702702702702704,
        "noul": 33.33333333333333,
        "score": 47.31695197960376
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 27.7843293385466
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5546666666666666,
       "cc": 42.30437035757471,
       "cc_by_type": {
        "choice": 42.30437035757471
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 42.30437035757471
      },
      "noul": {
       "accuracy": 0.408,
       "cc": -18.400000000000006,
       "cc_by_type": {
        "noul": -18.400000000000006
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.4026666666666667,
       "cc": 14.785081876637218,
       "cc_by_type": {
        "score": 14.785081876637218
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 14.785081876637218
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4166666666666667,
       "cc": -21.322066560908976,
       "cc_by_type": {
        "choice": 50.57660626029654,
        "noul": -60,
        "score": -54.542805943023474
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -20.883079422436126,
       "cc_by_type": {
        "choice": 1.8003273322422266,
        "noul": -19.999999999999996,
        "score": -44.4495655995506
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.55,
       "cc": 24.46389210511948,
       "cc_by_type": {
        "choice": 83.87096774193549,
        "noul": -50,
        "score": 39.52070857342296
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 24.46389210511948
      },
      "demand_forecasting": {
       "accuracy": 0.23076923076923078,
       "cc": -33.81567016039666,
       "cc_by_type": {
        "choice": -8.43373493975904,
        "noul": -100,
        "score": 6.986724458569061
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.3888888888888889,
       "cc": 4.454876806713197,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -60,
        "score": 29.265251538152015
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 4.454876806713197
      },
      "feature_extraction": {
       "accuracy": 0.3103448275862069,
       "cc": -6.573360507662842,
       "cc_by_type": {
        "choice": 5.9829059829059785,
        "noul": -19.999999999999996,
        "score": -5.702987505894508
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.46153846153846156,
       "cc": 16.612884611673458,
       "cc_by_type": {
        "choice": 47.82608695652174,
        "noul": 25,
        "score": -22.987433121501375
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 16.612884611673458
      },
      "gaming": {
       "accuracy": 0.41379310344827586,
       "cc": -6.703885975467817,
       "cc_by_type": {
        "choice": 40.59405940594059,
        "noul": -71.42857142857143,
        "score": 10.722854096227385
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.21428571428571427,
       "cc": -33.44343074859378,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": -100,
        "score": -20.965212880701966
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.17391304347826086,
       "cc": -31.072633553703692,
       "cc_by_type": {
        "choice": -0.7633587786259586,
        "noul": -100,
        "score": 7.5454581175148805
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.32,
       "cc": -1.078959073185669,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -75,
        "score": 4.283447983695021
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.15384615384615385,
       "cc": -23.678295476921054,
       "cc_by_type": {
        "choice": -14.754098360655735,
        "noul": -100,
        "score": 43.719211929892566
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.4827586206896552,
       "cc": 15.868347401942737,
       "cc_by_type": {
        "choice": 35.77981651376147,
        "noul": 25,
        "score": -13.174774307933257
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 15.868347401942737
      },
      "model_routing": {
       "accuracy": 0.6866666666666666,
       "cc": 57.504158219489966,
       "cc_by_type": {
        "choice": 63.012597158938625,
        "noul": 59.09090909090908,
        "score": 50.40896840862219
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 57.504158219489966
      },
      "moderation_trust_safety": {
       "accuracy": 0.4375,
       "cc": 7.985104053150214,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -60,
        "score": 47.113206896292745
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 7.985104053150214
      },
      "recruiting": {
       "accuracy": 0.38095238095238093,
       "cc": 0.5006746462399576,
       "cc_by_type": {
        "choice": 39.69849246231156,
        "noul": -66.66666666666667,
        "score": 28.470198143074978
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 0.5006746462399576
      },
      "risk_assessment": {
       "accuracy": 0.46153846153846156,
       "cc": 6.972667050442593,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": -33.333333333333336,
        "score": 10.855108069566777
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 6.972667050442593
      },
      "scientific_discovery": {
       "accuracy": 0.25,
       "cc": -22.282698718421653,
       "cc_by_type": {
        "choice": 12.350597609561751,
        "noul": -71.42857142857143,
        "score": -7.77012233625527
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.45454545454545453,
       "cc": -9.604825720085183,
       "cc_by_type": {
        "choice": 57.04057279236276,
        "noul": -100,
        "score": 14.144950047381688
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      70.92554914167138,
      79.78637653206756
     ],
     "I": [
      18.019293258402904,
      25.188069370363653
     ],
     "capability": [
      45.240972947803556,
      51.6024129927801
     ],
     "composite": {
      "A": [
       5.217354515822397,
       12.201151252166042
      ],
      "B": [
       4.192329175683889,
       10.346538617725733
      ],
      "C": [
       3.6231628582099975,
       8.473021702893085
      ]
     },
     "seed": 24
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 76.87411794897184,
     "capability": 49.50665640388703,
     "intelligence": 22.13919485880221
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5546666666666666,
      "calibration": 77.37204129541479,
      "cc": 42.30437035757471,
      "cc_by_type": {
       "choice": 42.30437035757471
      },
      "cc_tiered": 46.38805695454458,
      "cc_tiered_open": 53.72008003683186,
      "cc_tiered_sealed": 39.056033872257295,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 42.30437035757471
     },
     "noul": {
      "accuracy": 0.408,
      "calibration": 77.27920881601906,
      "cc": -18.400000000000006,
      "cc_by_type": {
       "noul": -18.400000000000006
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4026666666666667,
      "calibration": 75.9711037354817,
      "cc": 14.785081876637218,
      "cc_by_type": {
       "score": 14.785081876637218
      },
      "cc_tiered": 20.02952762186206,
      "cc_tiered_open": 29.847390838405925,
      "cc_tiered_sealed": 10.211664405318189,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 14.785081876637218
     }
    },
    "raw_on_A": {
     "calibration": 72.6387411013128,
     "capability": 46.47345221333068,
     "intelligence": 20.308163325348563
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 7.7826427506909805,
    "Emphasis on Accuracy 60:20:20": 5.908464633435587,
    "Emphasis on Cost 20:20:60": 8.514539745568566,
    "Emphasis on Speed 20:60:20": 10.123680376098372,
    "Intelligence only": 4.340556958274228
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Liquid AI",
   "axes": {
    "calibration": 62.32062493293682,
    "cost": 73.1039158657426,
    "intelligence": 3.2615852143588597,
    "speed": 93.99244100826726
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.22880094970871764,
      "ece_score": 54.23981005825647,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 54.23981005825647,
      "score_before_time_limit": 54.23981005825647,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.7333333333333334,
      "brier": 0.2769707015707441,
      "ece": 0.1533565516303061,
      "n": 375,
      "score": 69.32868967393878,
      "score_before_time_limit": 69.32868967393878,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8293333333333334,
      "interval90_width": 0.6586666666666664,
      "n": 375,
      "nrps": 0.21825554880108286,
      "score": 63.39337506661522,
      "score_before_time_limit": 63.39337506661522,
      "time_limit_failed_share": 0,
      "top_ece": 0.25693847493330635
     }
    },
    "score": 62.32062493293682
   },
   "capability": 32.79110507364784,
   "composite_ci95": {
    "A": [
     0.007928751847345473,
     0.33896544733805034
    ],
    "B": [
     0.005122646508709089,
     0.23614958881345818
    ],
    "C": [
     0.005506077671767688,
     0.23539267176253492
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no bookable listing for this base. The declared text base LiquidAI/LFM2.5-Encoder-350M is not a key of any frozen pricing table. Method as above: deepinfra:Qwen/Qwen3.5-0.8B at USD 0.01 per 1M input and USD 0.05 per 1M output - the class above this 600M model - applied to its own measured usage, 1,182,027 input tokens and 0 output tokens over 1,500 decisions. The frozen base-size ENCODER rate (0.005, 0.0), which the published tacet-sonata and lev-350m rows carry, would give 0.00394009 and was deliberately NOT used: it prices a text-only encoder, while d1-omni also runs SigLIP2 vision and FastConformer audio towers, so the class above is the conservative choice and keeps the two Liquid rows on one method.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00788018
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Liquid AI d1-omni-600M",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX A6000 48 GB pod (United States), offline after setup (egress cut and proven), author's custom code at the pinned commit via trust_remote_code from the sha256-verified local copy, float16, states cut on the right at 16,384 tokens by the model code",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 4.293220672965477,
    "I_sealed": 2.229949755752243,
    "base": 3.2615852143588597,
    "excess": -8.009973664126479,
    "gap": 2.063270917213234,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 12.87966201889643,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486504,
       "hard": 3.743315508021402,
       "judge": 12.986156888595916,
       "standard": 32.18920050230222
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -73.91304347826086,
       "judge": -100,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 12.092551202372192,
       "hard": -10.469003850745073,
       "judge": 19.089799023912114,
       "standard": -20.24343611383641
      }
     },
     "sealed|choice": {
      "cc": 6.689849267256729,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 2.388153092084683,
       "hard": 3.5396284489392005,
       "judge": 5.9408452133326675,
       "standard": 16.2646450723639
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": -64.1025641025641,
       "hard": -72.72727272727273,
       "judge": -72.22222222222221,
       "standard": -64.91228070175438
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": -14.21064395948739,
       "hard": -26.382300409685588,
       "judge": -13.560363505639916,
       "standard": -9.691900455584058
      }
     }
    }
   },
   "jevbench_score": 0.04905613733213806,
   "key": "liquid-d1-omni-600m",
   "last_measured_on": "2026-10-10",
   "licence": "LFM Open License v1.0 (commercial use only below USD 10 M annual revenue)",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "LiquidAI/d1-omni-600M@02b55d7076f15129e59ab3f94783f32c4b088674",
   "noul_decisive": {
    "acc_among_decisive": 0.56,
    "decisive": 100,
    "decisive_rate": 0.26666666666666666,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 62.32062493293682,
     "cost": 73.1039158657426,
     "intelligence": 3.2615852143588597,
     "speed": 94.0744546594334
    },
    "board_release": "v1.6.43",
    "capability": 32.79110507364784,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 126,
    "capability_rank": 155,
    "composite_ci95": {
     "A": [
      0.00792887171051229,
      0.3389815219491588
     ],
     "B": [
      0.005122686535585832,
      0.23615583025477924
     ],
     "C": [
      0.005506160910077979,
      0.23540383468691578
     ]
    },
    "jevbench_score": 0.049057448753865106,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 182,
    "rank": 226,
    "ranks": {
     "A": 226,
     "B": 226,
     "C": 226,
     "capability": 222
    },
    "scores": {
     "A": 0.049057448753865106,
     "B": 0.032554129042754114,
     "C": 0.03406767274573965
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19256855882704257,
     "p50_s_raw": 0.02128427941352129,
     "p95_s_adjusted": 0.20322126662358642,
     "p95_s_raw": 0.026610633311793207
    },
    "views": {
     "Balanced 33:33:33": 0.038577217040809395,
     "Emphasis on Accuracy 60:20:20": 0.022535446033063476,
     "Emphasis on Cost 20:20:60": 0.05938557623710964,
     "Emphasis on Speed 20:60:20": 0.06041396685797389,
     "Intelligence only": 0.013878616661020313
    }
   },
   "rank": 268,
   "ranked": true,
   "ranks": {
    "A": 268,
    "B": 268,
    "C": 268,
    "capability": 263
   },
   "repo": "https://huggingface.co/LiquidAI/d1-omni-600M",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.04905613733213806,
    "B": 0.032553667044800734,
    "C": 0.03406676203620698
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19325650818645954,
    "p50_s_raw": 0.02162825409322977,
    "p95_s_adjusted": 0.20635821893811224,
    "p95_s_raw": 0.028179109469056128
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.20689655172413793,
       "cc": -29.121942106010263,
       "cc_by_type": {
        "choice": -8.333333333333334,
        "noul": -100,
        "score": 20.967507015302544
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.1875,
       "cc": -48.670081675280734,
       "cc_by_type": {
        "choice": -0.1552795031055859,
        "noul": -76.92307692307692,
        "score": -68.93188859965969
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.22448979591836735,
       "cc": -44.09414082419346,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -78.57142857142857,
        "score": -64.82210501226291
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.2222222222222222,
       "cc": -39.3458024428534,
       "cc_by_type": {
        "choice": 29.411764705882355,
        "noul": -100,
        "score": -47.44917203444254
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.2918918918918919,
       "cc": -18.904988531894233,
       "cc_by_type": {
        "choice": 11.069939786938404,
        "noul": -64.1025641025641,
        "score": -3.682341280057
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.3446969696969697,
       "cc": 1.1432998727522918,
       "cc_by_type": {
        "choice": 15.466790524849063,
        "noul": -28.767123287671236,
        "score": 16.73023238107905
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 1.1432998727522918
      },
      "long_state": {
       "accuracy": 0.27586206896551724,
       "cc": -25.065654575351683,
       "cc_by_type": {
        "choice": 16.66666666666667,
        "noul": -100,
        "score": 8.136369607278283
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.22580645161290322,
       "cc": -33.79734650736005,
       "cc_by_type": {
        "choice": 7.692307692307688,
        "noul": -100,
        "score": -9.084347214387822
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.2,
       "cc": -31.03237123270632,
       "cc_by_type": {
        "choice": -1.2091356918943095,
        "noul": -93.10344827586206,
        "score": 1.2154702696374198
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.16071428571428573,
       "cc": -85.43494727513347,
       "cc_by_type": {
        "choice": -4.761904761904764,
        "noul": -90,
        "score": -161.54293706349563
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.2807017543859649,
       "cc": 8.99001109877914,
       "cc_by_type": {
        "choice": 8.99001109877914
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 8.99001109877914
      },
      "perturb_pair": {
       "accuracy": 0.31666666666666665,
       "cc": -11.077733022650753,
       "cc_by_type": {
        "choice": 21.36304062909567,
        "noul": -85.71428571428572,
        "score": 31.1180460172378
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.2320675105485232,
       "cc": -25.394686423610008,
       "cc_by_type": {
        "choice": 5.184043168240532,
        "noul": -57.89473684210527,
        "score": -23.47336559696529
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.09090909090909091,
       "cc": -53.76070940660107,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": -27.948794886469862
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0,
       "cc": -63.559322033898304,
       "cc_by_type": {
        "choice": -27.11864406779661,
        "noul": -100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.375,
       "cc": -22.128997159813196,
       "cc_by_type": {
        "choice": 17.159763313609464,
        "noul": -100,
        "score": 16.45324520695096
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.26666666666666666,
       "cc": -26.699211006624235,
       "cc_by_type": {
        "choice": 7.6644538322269185,
        "noul": -77.77777777777779,
        "score": -9.984309074321839
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 0
      },
      "el": {
       "accuracy": 0.16666666666666666,
       "cc": -44.641658892653076,
       "cc_by_type": {
        "choice": -8.43373493975904,
        "noul": -100,
        "score": -25.491241738200188
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.26273458445040215,
       "cc": -24.234183293883643,
       "cc_by_type": {
        "choice": 8.693213684800824,
        "noul": -62.40601503759399,
        "score": -18.989748528857753
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.21428571428571427,
       "cc": -23.875465734890014,
       "cc_by_type": {
        "choice": 8.363636363636363,
        "noul": -100,
        "score": 20.00996643169359
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.2222222222222222,
       "cc": -42.25352112676057,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": -100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.2,
       "cc": -32.05973275794293,
       "cc_by_type": {
        "choice": -1.5514809590973198,
        "noul": -100,
        "score": 5.372282685268514
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.2,
       "cc": -27.81219448998868,
       "cc_by_type": {
        "choice": 13.043478260869566,
        "noul": -100,
        "score": 3.519938269164402
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.16666666666666666,
       "cc": -48.387096774193544,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.23529411764705882,
       "cc": -47.53372129710758,
       "cc_by_type": {
        "choice": 23.83638928067701,
        "noul": -100,
        "score": -66.43755317199978
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.125,
       "cc": -47.14183058522813,
       "cc_by_type": {
        "choice": -13.821138211382111,
        "noul": -100,
        "score": -27.604353544302263
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.18181818181818182,
       "cc": -43.344897587116954,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100,
        "score": 1.1128482222557023
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.45454545454545453,
       "cc": 16.030320645251987,
       "cc_by_type": {
        "choice": 56.04395604395604,
        "noul": -50,
        "score": 42.04700589179993
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 16.030320645251987
      },
      "nl": {
       "accuracy": 0.23076923076923078,
       "cc": -20.35052023493665,
       "cc_by_type": {
        "choice": 24.79108635097493,
        "noul": -66.66666666666667,
        "score": -19.17598038911821
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.14285714285714285,
       "cc": -49.18032786885246,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.225,
       "cc": -12.618486802534166,
       "cc_by_type": {
        "choice": 5.553126366418887,
        "noul": -60,
        "score": 16.591413225978613
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.2222222222222222,
       "cc": -28.382395955071996,
       "cc_by_type": {
        "choice": 9.547738693467334,
        "noul": -66.66666666666667,
        "score": -28.02825989201665
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0,
       "cc": -48.522135284211906,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": -12.233072519302368
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.36363636363636365,
       "cc": -18.188721011256614,
       "cc_by_type": {
        "choice": -5.26315789473684,
        "noul": -100,
        "score": 50.696994860967
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.09090909090909091,
       "cc": -41.07722171240693,
       "cc_by_type": {
        "choice": -30.43478260869565,
        "noul": -100,
        "score": 7.2031174714748625
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.375,
       "cc": -10.272994610640382,
       "cc_by_type": {
        "choice": 35.13513513513514,
        "noul": -100,
        "score": 34.045881032943726
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.2853333333333333,
       "cc": 7.410606322335463,
       "cc_by_type": {
        "choice": 7.410606322335463
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 7.410606322335463
      },
      "noul": {
       "accuracy": 0.14933333333333335,
       "cc": -70.13333333333334,
       "cc_by_type": {
        "noul": -70.13333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.2853333333333333,
       "cc": -14.348760038030782,
       "cc_by_type": {
        "score": -14.348760038030782
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2916666666666667,
       "cc": -18.14065897262343,
       "cc_by_type": {
        "choice": -8.731466227347617,
        "noul": -19.999999999999996,
        "score": -25.690510690522682
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.16666666666666666,
       "cc": -19.845708892643234,
       "cc_by_type": {
        "choice": -8.01963993453355,
        "noul": -60,
        "score": 8.482513256603841
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.2,
       "cc": -36.093708917165806,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -75,
        "score": -36.506933203110336
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.19230769230769232,
       "cc": -43.411243099056556,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -100,
        "score": -45.89637989957931
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2222222222222222,
       "cc": -19.47483978171899,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -60,
        "score": -5.256817481802933
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.10344827586206896,
       "cc": -52.342876150533066,
       "cc_by_type": {
        "choice": -11.111111111111114,
        "noul": -100,
        "score": -45.917517340488075
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.38461538461538464,
       "cc": 0.6176093078177957,
       "cc_by_type": {
        "choice": 8.695652173913043,
        "noul": 0,
        "score": -6.842824250459656
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 0.6176093078177957
      },
      "gaming": {
       "accuracy": 0.20689655172413793,
       "cc": -26.6661863971574,
       "cc_by_type": {
        "choice": 16.831683168316832,
        "noul": -71.42857142857143,
        "score": -25.40167093121759
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.2857142857142857,
       "cc": 4.412051451900017,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": -100,
        "score": 92.60123372077942
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 4.412051451900017
      },
      "knowledge_graphs": {
       "accuracy": 0.08695652173913043,
       "cc": -55.71206240218595,
       "cc_by_type": {
        "choice": -9.923664122137408,
        "noul": -100,
        "score": -57.21252308442044
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.28,
       "cc": -16.228836402216572,
       "cc_by_type": {
        "choice": 18.69918699186992,
        "noul": -100,
        "score": 32.61430380148036
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.23076923076923078,
       "cc": -29.703887276787327,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": 9.248993907342928
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.3103448275862069,
       "cc": -15.934116861404396,
       "cc_by_type": {
        "choice": 17.431192660550458,
        "noul": -50,
        "score": -15.233543244763649
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.4,
       "cc": 4.006137375452837,
       "cc_by_type": {
        "choice": 27.633342267488622,
        "noul": -40.90909090909091,
        "score": 25.294160767960793
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 4.006137375452837
      },
      "moderation_trust_safety": {
       "accuracy": 0.25,
       "cc": -29.448869234338442,
       "cc_by_type": {
        "choice": 5.263157894736842,
        "noul": -100,
        "score": 6.390234402247841
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.23809523809523808,
       "cc": -26.169667953406684,
       "cc_by_type": {
        "choice": 24.62311557788945,
        "noul": -100,
        "score": -3.132119438109493
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.23076923076923078,
       "cc": -41.00485732486038,
       "cc_by_type": {
        "choice": 5.660377358490565,
        "noul": -100,
        "score": -28.67494933307171
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.15625,
       "cc": -22.981094893304945,
       "cc_by_type": {
        "choice": -11.553784860557771,
        "noul": -71.42857142857143,
        "score": 14.03907160921436
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.13636363636363635,
       "cc": -52.2527400518963,
       "cc_by_type": {
        "choice": -14.558472553699291,
        "noul": -100,
        "score": -42.199747601989635
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      56.7149821052611,
      66.00061446604302
     ],
     "I": [
      1.7436767620361024,
      6.437188407255578
     ],
     "capability": [
      29.712243529714982,
      35.427303807623055
     ],
     "composite": {
      "A": [
       0.007928751847345473,
       0.33896544733805034
      ],
      "B": [
       0.005122646508709089,
       0.23614958881345818
      ],
      "C": [
       0.005506077671767688,
       0.23539267176253492
      ]
     },
     "seed": 25
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 62.32062493293682,
     "capability": 32.79110507364784,
     "intelligence": 3.2615852143588597
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2853333333333333,
      "calibration": 54.23981005825647,
      "cc": 7.410606322335463,
      "cc_by_type": {
       "choice": 7.410606322335463
      },
      "cc_tiered": 9.78475564307658,
      "cc_tiered_open": 12.87966201889643,
      "cc_tiered_sealed": 6.689849267256729,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 7.410606322335463
     },
     "noul": {
      "accuracy": 0.14933333333333335,
      "calibration": 69.32868967393878,
      "cc": -70.13333333333334,
      "cc_by_type": {
       "noul": -70.13333333333334
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2853333333333333,
      "calibration": 63.39337506661522,
      "cc": -14.348760038030782,
      "cc_by_type": {
       "score": -14.348760038030782
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 60.8515976034804,
     "capability": 31.49910396998157,
     "intelligence": 2.1466103364827385
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.03857613577512699,
    "Emphasis on Accuracy 60:20:20": 0.022535224640971675,
    "Emphasis on Cost 20:20:60": 0.05938403884460177,
    "Emphasis on Speed 20:60:20": 0.06040919381107833,
    "Intelligence only": 0.013878616661020313
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Liquid AI",
   "axes": {
    "calibration": 91.25367387086692,
    "cost": 62.300793859384434,
    "intelligence": 69.70041168706942,
    "speed": 88.94430045323229
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.038374286188809156,
      "ece_score": 92.32514276223817,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 92.32514276223817,
      "score_before_time_limit": 92.32514276223817,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.14477211796246647,
      "brier": 0.0787106673820862,
      "ece": 0.060856253290503357,
      "n": 373,
      "score": 87.82874934189933,
      "score_before_time_limit": 87.82874934189933,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.26276881720430106,
      "n": 372,
      "nrps": 0.05673524666969873,
      "score": 93.60712950846329,
      "score_before_time_limit": 93.60712950846329,
      "time_limit_failed_share": 0,
      "top_ece": 0.03556108158051766
     }
    },
    "score": 91.25367387086692
   },
   "capability": 80.47704277896817,
   "composite_ci95": {
    "A": [
     73.89351069961306,
     76.97573052365466
    ],
    "B": [
     71.94286014570793,
     76.23436422046282
    ],
    "C": [
     73.89351069961306,
     76.97573052365466
    ]
   },
   "cost": {
    "basis": "Liquid AI public price list, USD 0.042/M input, output free (fetched 16 Oct 2026): the models table of Liquid's public pricing page lists d1-ultra at USD 0.042 per 1M input tokens and USD 0.00 per 1M output tokens (saved copy sha256 89fb8ef2...); the same rate Liquid's API billed on the measurement's smoke call (48 input tokens -> USD 0.000002016). Counted as the manufacturer's standard launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops and SubQ Snap), through a scoped re-freeze of the frozen pricing module that adds this one key and nothing else, with a zero-difference proof for every other registry entry. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (638,436 input tokens over the 1,485 answered items, 0 output tokens), divided by the 1,485 items of the common cost basis the scorer uses; the 15 items refused with HTTP 422 count as zero tokens and as wrong answers. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.018057 per 1,000 decisions (0.280x the official cost cap USD 0.064595), against USD 0.017876 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.017876208
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01805677575757576
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "d1-ultra (Liquid AI)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-15",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.liquid.ai/decisions/v1/systemone API, model `d1-ultra`, through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `liquid` before the first request (approved 15 Oct 2026). Native probabilities on all three question types. 15 of 1,500 items exceed the model's 32,768-token input limit (HTTP 422) and are scored as failures.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 75.72153851823367,
    "I_sealed": 63.679284855905166,
    "base": 69.70041168706942,
    "excess": 2.2027268919727945,
    "gap": 12.042253662328505,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 88.89079890008239,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 88.13447593935399,
       "standard": 100
      }
     },
     "open|noul": {
      "cc": 75.17046238785369,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 63.10335426676494,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.60054737282905,
       "hard": 68.18905752206545,
       "judge": 75.94709495533684,
       "standard": 35.917740170274
      }
     },
     "sealed|choice": {
      "cc": 79.10284436684508,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 72.93859141898925,
       "judge": 79.72575905974536,
       "standard": 84.81333052098715
      }
     },
     "sealed|noul": {
      "cc": 53.0885768888857,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 72.41379310344827,
       "hard": 43.636363636363626,
       "judge": 53.846153846153854,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 58.84643331198474,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 81.7986845210918,
       "hard": 56.171536782426415,
       "judge": 58.372812971884905,
       "standard": 53.43053127669758
      }
     }
    }
   },
   "jevbench_score": 76.04601536374408,
   "key": "liquid-d1-ultra",
   "last_measured_on": "2026-10-15",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.21",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.liquid.ai/decisions/v1/systemone#d1-ultra",
   "noul_decisive": {
    "acc_among_decisive": 0.9373040752351097,
    "decisive": 319,
    "decisive_rate": 0.8552278820375335,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 3,
    "axes": {
     "calibration": 91.25367387086692,
     "cost": 62.300793859384434,
     "intelligence": 69.70041168706942,
     "speed": 88.99977436791615
    },
    "board_release": "v1.6.43",
    "capability": 80.47704277896817,
    "capability_api_board_rank": 6,
    "capability_open_board_rank": null,
    "capability_rank": 11,
    "composite_ci95": {
     "A": [
      73.90307802669203,
      76.98611269050402
     ],
     "B": [
      71.95011503809144,
      76.24251050830729
     ],
     "C": [
      73.90307802669203,
      76.98611269050402
     ]
    },
    "jevbench_score": 76.05614823638027,
    "measured_in": "v1.6.21",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 4,
    "ranks": {
     "A": 4,
     "B": 5,
     "C": 4,
     "capability": 22
    },
    "scores": {
     "A": 76.05614823638027,
     "B": 74.69393258837212,
     "C": 76.05614823638027
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.2313630650169216,
     "p50_s_raw": 0.2313630650169216,
     "p95_s_adjusted": 0.5441624051978579,
     "p95_s_raw": 0.5441624051978579
    },
    "views": {
     "Balanced 33:33:33": 72.05604070021411,
     "Emphasis on Accuracy 60:20:20": 71.0949370716979,
     "Emphasis on Cost 20:20:60": 67.80894855156113,
     "Emphasis on Speed 20:60:20": 77.99554201006464,
     "Intelligence only": 69.70041168706942
    }
   },
   "rank": 9,
   "ranked": true,
   "ranks": {
    "A": 9,
    "B": 16,
    "C": 8,
    "capability": 32
   },
   "repo": "https://www.liquid.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 76.04601536374408,
    "B": 74.68611384483172,
    "C": 76.04601536374408
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2306590935040731,
    "p50_s_raw": 0.2306590935040731,
    "p95_s_adjusted": 0.5528398923954227,
    "p95_s_raw": 0.5528398923954227
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 80.02551491048317,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 48.40987806478281
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 80.02551491048317
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 53.5127160507667,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": 60.00000000000001,
        "score": 45.07685440892786
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 53.5127160507667
      },
      "composite_gen": {
       "accuracy": 0.9306930693069307,
       "cc": 89.03033227085523,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 81.00246926062347
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 89.03033227085523
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.90669247576356,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 97.72007742729068
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.90669247576356
      },
      "jaggedness": {
       "accuracy": 0.7111111111111111,
       "cc": 52.548678213136746,
       "cc_by_type": {
        "choice": 62.463907603464875,
        "noul": 35.13513513513513,
        "score": 60.04699190081021
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 52.548678213136746
      },
      "llm_router": {
       "accuracy": 0.9041533546325878,
       "cc": 80.3197048932887,
       "cc_by_type": {
        "choice": 91.28290921898389,
        "noul": 89.69072164948453,
        "score": 59.98548381139766
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 80.3197048932887
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 40.01561610820875,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 0,
        "score": 44.789116365863336
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 40.01561610820875
      },
      "multilingual": {
       "accuracy": 0.7418181818181818,
       "cc": 55.53031034406329,
       "cc_by_type": {
        "choice": 80.43690903162701,
        "noul": 25,
        "score": 61.15402200056285
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 55.53031034406329
      },
      "negation_pair_gen": {
       "accuracy": 0.9302325581395349,
       "cc": 76.76565063912945,
       "cc_by_type": {
        "choice": 100,
        "noul": 80,
        "score": 50.296951917388334
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 76.76565063912945
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 80.13869805419485,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 52.42810617459654
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.13869805419485
      },
      "usecase": {
       "accuracy": 0.7681818181818182,
       "cc": 61.12254914211481,
       "cc_by_type": {
        "choice": 80.39215686274511,
        "noul": 41.66666666666667,
        "score": 61.30882389693265
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 61.12254914211481
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 27.790054081448684,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 0,
        "score": 2.417781291965093
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 27.790054081448684
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 70.84904875775025,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 100,
        "score": 54.021800651130945
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 70.84904875775025
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 89.08391810246775,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 83.18095784722625
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 89.08391810246775
      },
      "el": {
       "accuracy": 0.6,
       "cc": 44.35440994419011,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -33.333333333333336,
        "score": 99.91611623852936
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.35440994419011
      },
      "en": {
       "accuracy": 0.8236316246741964,
       "cc": 69.98283524145941,
       "cc_by_type": {
        "choice": 80.88868820248715,
        "noul": 67.62589928057554,
        "score": 61.433918241315546
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 69.98283524145941
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 66.08416705224533,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 100,
        "score": 27.616285822315124
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.08416705224533
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 76.59925311455045,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 96.46442601031806
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 76.59925311455045
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 96.12745113668352,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 88.38235341005056
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 96.12745113668352
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 69.2291292495233,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 50,
        "score": 57.68738774856992
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 69.2291292495233
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 62.7322379583571,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -11.803286124928691
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 62.7322379583571
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 24.203419893141085,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 0,
        "score": -13.092679160926245
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 24.203419893141085
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 18.422112493954483,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 55.26633748186346
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 18.422112493954483
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.4730866883171,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.41926006495127
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.4730866883171
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -5.121222995410733,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 10.610356987793757
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 66.50566804059815,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 67.13443958518846
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 66.50566804059815
      },
      "pt": {
       "accuracy": 0.8571428571428571,
       "cc": 85.83933080677154,
       "cc_by_type": {
        "choice": 89.39304655274012,
        "noul": 100,
        "score": 68.1249458675745
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 85.83933080677154
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 45.882356110583025,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 19.999999999999996,
        "score": 43.17898322536608
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 45.882356110583025
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 47.56476228162157,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0,
        "score": 64.43341727964732
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 47.56476228162157
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 78.73277592080343,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 68.45639227853934
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 78.73277592080343
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 46.87044056187991,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 7.277988352306409
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.87044056187991
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8546666666666667,
       "cc": 81.22811381771561,
       "cc_by_type": {
        "choice": 81.22811381771561
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.22811381771561
      },
      "noul": {
       "accuracy": 0.7973333333333333,
       "cc": 59.46666666666667,
       "cc_by_type": {
        "noul": 59.46666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 59.46666666666667
      },
      "score": {
       "accuracy": 0.7333333333333333,
       "cc": 60.44867908600179,
       "cc_by_type": {
        "score": 60.44867908600179
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 60.44867908600179
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 33.23171209119251,
       "cc_by_type": {
        "choice": 84,
        "noul": -25,
        "score": 40.695136273577546
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 33.23171209119251
      },
      "code_linting": {
       "accuracy": 0.9583333333333334,
       "cc": 94.09748592551544,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 82.29245777654631
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 94.09748592551544
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 62.92582030524064,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100,
        "score": 37.557948720599974
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 62.92582030524064
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": -17.965299307538217,
       "cc_by_type": {
        "choice": 64.91228070175438,
        "noul": -100,
        "score": -18.80817862436903
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 24.49212514702329,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 64.78072326715683
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 24.49212514702329
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": -13.88384228808446,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": -124.67039478878168
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 35.72504937468765,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": -42.85714285714286,
        "score": 59.83621254983327
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 35.72504937468765
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 31.77084188905293,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -50,
        "score": 76.6714106845804
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 31.77084188905293
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 45.1815821381559,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 19.999999999999996,
        "score": 57.211413081134374
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 45.1815821381559
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 76.48497795483895,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 62.7882671978502
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 76.48497795483895
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 66.48644556865014,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 33.309465904916834
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 66.48644556865014
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -11.843346976830077,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": 26.835550467359237
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 55.32046425195082,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 45.96139275585243
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 55.32046425195082
      },
      "model_routing": {
       "accuracy": 0.9078341013824884,
       "cc": 79.30093337735978,
       "cc_by_type": {
        "choice": 96.49350649350649,
        "noul": 93.93939393939394,
        "score": 47.46989969917894
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.30093337735978
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 20.596744504671886,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 84.82286307255691
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 20.596744504671886
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.36102452879326,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 63.17592409705759
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.36102452879326
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 17.81076588752592,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50,
        "score": 18.21062278573048
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 17.81076588752592
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 24.11187241023282,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": -66.66666666666667,
        "score": 53.495037520553545
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 24.11187241023282
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.89828165670299,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.69484497010896
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.89828165670299
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.25252044978473,
      91.63978255866996
     ],
     "I": [
      64.56901745362967,
      73.4726823695734
     ],
     "capability": [
      76.18500946956924,
      81.95767090067203
     ],
     "composite": {
      "A": [
       73.89351069961306,
       76.97573052365466
      ],
      "B": [
       71.94286014570793,
       76.23436422046282
      ],
      "C": [
       73.89351069961306,
       76.97573052365466
      ]
     },
     "seed": 45
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.25367387086692,
     "capability": 80.47704277896817,
     "intelligence": 69.70041168706942
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8546666666666667,
      "calibration": 92.32514276223817,
      "cc": 81.22811381771561,
      "cc_by_type": {
       "choice": 81.22811381771561
      },
      "cc_tiered": 83.99682163346372,
      "cc_tiered_open": 88.89079890008239,
      "cc_tiered_sealed": 79.10284436684508,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.22811381771561
     },
     "noul": {
      "accuracy": 0.7973333333333333,
      "calibration": 87.82874934189933,
      "cc": 59.46666666666667,
      "cc_by_type": {
       "noul": 59.46666666666667
      },
      "cc_tiered": 64.12951963836969,
      "cc_tiered_open": 75.17046238785369,
      "cc_tiered_sealed": 53.0885768888857,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 59.46666666666667
     },
     "score": {
      "accuracy": 0.7333333333333333,
      "calibration": 93.60712950846329,
      "cc": 60.44867908600179,
      "cc_by_type": {
       "score": 60.44867908600179
      },
      "cc_tiered": 60.97489378937484,
      "cc_tiered_open": 63.10335426676494,
      "cc_tiered_sealed": 58.84643331198474,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 60.44867908600179
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 72.0439144024908,
    "Emphasis on Accuracy 60:20:20": 71.08785360500364,
    "Emphasis on Cost 20:20:60": 67.80250471482547,
    "Emphasis on Speed 20:60:20": 77.96997204149442,
    "Intelligence only": 69.70041168706942
   }
  },
  {
   "adapter_id": "images/litjev/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Zhengxu Yu",
   "axes": {
    "calibration": 89.9210925231847,
    "cost": 55.72786321561281,
    "intelligence": 43.28549422942267,
    "speed": 68.06914373137408
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.054566021834308726,
      "ece_score": 89.08679563313827,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 89.08679563313827,
      "score_before_time_limit": 89.08679563313827,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.33692722371967654,
      "brier": 0.09143307684062531,
      "ece": 0.03620776305464309,
      "n": 371,
      "score": 92.75844738907139,
      "score_before_time_limit": 92.75844738907139,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.981029810298103,
      "interval90_width": 0.534327009936766,
      "n": 369,
      "nrps": 0.08453630674044608,
      "score": 87.91803454734445,
      "score_before_time_limit": 87.91803454734445,
      "time_limit_failed_share": 0,
      "top_ece": 0.07855150115633251
     }
    },
    "score": 89.9210925231847
   },
   "capability": 66.60329337630368,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 944,931 input + 0 output tokens over 1,624 decisions = 581.854064039 tokens/decision. GPU-time cost USD 0.0299045927762475/1,000; old list-price cost USD 0.244378706896552/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0299045927762475/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.029904592776247543
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "LitJev (Qwen3.8-27B)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 41.98760951977819,
    "I_sealed": 44.583378939067146,
    "base": 43.28549422942267,
    "excess": -5.1691570617272,
    "gap": -2.595769419288956,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 68.57906399743361,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 64.1396665618119,
       "judge": 68.35860250494396,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 12.532781228433404,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 4.347826086956519,
       "judge": 0,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 44.85098333346756,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.30288571369996,
       "hard": 36.97174671674506,
       "judge": 54.42696015026953,
       "standard": 39.519540151593404
      }
     },
     "sealed|choice": {
      "cc": 68.65215618863985,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 60.29411764705882,
       "judge": 66.4804469273743,
       "standard": 82.26382559428504
      }
     },
     "sealed|noul": {
      "cc": 24.95501594133015,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": -0.9708737864077666,
       "judge": 28.947368421052634,
       "standard": 49.29577464788733
      }
     },
     "sealed|score": {
      "cc": 40.142964687231455,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 64.90525292252141,
       "hard": 32.24057461852555,
       "judge": 45.24652806264088,
       "standard": 35.91125564388415
      }
     }
    }
   },
   "jevbench_score": 44.83814187422011,
   "key": "litjev",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (code); Apache-2.0 base weights",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.975609756097561,
    "decisive_rate": 0.6630727762803235,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.9210925231847,
     "cost": 28.35809912639418,
     "intelligence": 43.28549422942267,
     "speed": 68.17671225113584
    },
    "board_release": "v1.6.43",
    "capability": 66.60329337630368,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      8.746773766553591,
      14.766757637183968
     ],
     "B": [
      8.426311974704314,
      14.74912210872967
     ],
     "C": [
      6.074148448995549,
      10.25469280359998
     ]
    },
    "jevbench_score": 11.458886170296882,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 126,
    "rank": 165,
    "ranks": {
     "A": 165,
     "B": 164,
     "C": 166,
     "capability": 95
    },
    "scores": {
     "A": 11.458886170296882,
     "B": 11.238393678896292,
     "C": 7.957559840483947
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 3.0960148913785814,
     "p50_s_raw": 1.4730074456892908,
     "p95_s_adjusted": 4.9150250315666195,
     "p95_s_raw": 2.3825125157833096
    },
    "views": {
     "Balanced 33:33:33": 9.902804220753023,
     "Emphasis on Accuracy 60:20:20": 10.109112973826948,
     "Emphasis on Cost 20:20:60": 8.396442160471732,
     "Emphasis on Speed 20:60:20": 11.774987261719835,
     "Intelligence only": 10.435213933727047
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 130,
   "ranked": true,
   "ranks": {
    "A": 130,
    "B": 132,
    "C": 135,
    "capability": 122
   },
   "repo": "https://github.com/zhengxuyu/litjev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 44.83814187422011,
    "B": 41.654358826315935,
    "C": 31.137598523763973
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 3.1454328150488435,
    "p50_s_raw": 1.4977164075244218,
    "p95_s_adjusted": 4.959126807423309,
    "p95_s_raw": 2.4045634037116543
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B, frozen, read at the output head",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.4774023239196,
      89.78804799226427
     ],
     "I": [
      38.555280103835365,
      48.498383980718195
     ],
     "capability": [
      62.48585734643129,
      68.14178334863401
     ],
     "composite": null,
     "seed": 61
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.9210925231847,
     "capability": 66.60329337630368,
     "intelligence": 43.28549422942267
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7613333333333333,
      "calibration": 89.08679563313827,
      "cc": 69.47794208197946,
      "cc_by_type": {
       "choice": 69.47794208197946
      },
      "cc_tiered": 68.61561009303674,
      "cc_tiered_open": 68.57906399743361,
      "cc_tiered_sealed": 68.65215618863985,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.47794208197946
     },
     "noul": {
      "accuracy": 0.64,
      "calibration": 92.75844738907139,
      "cc": 28.000000000000004,
      "cc_by_type": {
       "noul": 28.000000000000004
      },
      "cc_tiered": 18.743898584881777,
      "cc_tiered_open": 12.532781228433404,
      "cc_tiered_sealed": 24.95501594133015,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 28.000000000000004
     },
     "score": {
      "accuracy": 0.5626666666666666,
      "calibration": 87.91803454734445,
      "cc": 41.34354803654359,
      "cc_by_type": {
       "score": 41.34354803654359
      },
      "cc_tiered": 42.496974010349504,
      "cc_tiered_open": 44.85098333346756,
      "cc_tiered_sealed": 40.142964687231455,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 41.34354803654359
     }
    },
    "raw_on_A": {
     "calibration": 89.32478284904047,
     "capability": 69.19315337175932,
     "intelligence": 49.06152389447817
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 40.33822942139412,
    "Emphasis on Accuracy 60:20:20": 36.758617497627675,
    "Emphasis on Cost 20:20:60": 40.89724952024707,
    "Emphasis on Speed 20:60:20": 44.0235595197873,
    "Intelligence only": 32.44046966302888
   }
  },
  {
   "adapter_id": "images/localjev-qwen3.5-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Amith Chandrappa (amithgc)",
   "axes": {
    "calibration": 85.84286362718517,
    "cost": 77.53686751699048,
    "intelligence": 24.145363218466134,
    "speed": 85.6437855976805
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.039143749999999984,
      "ece_score": 92.17125,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 92.17125,
      "score_before_time_limit": 92.17125,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.648,
      "brier": 0.14289767218666674,
      "ece": 0.09039973333333337,
      "n": 375,
      "score": 81.92005333333331,
      "score_before_time_limit": 81.92005333333331,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.96,
      "interval90_width": 0.6557777777777777,
      "n": 375,
      "nrps": 0.13420838236888896,
      "score": 83.4372875482222,
      "score_before_time_limit": 83.4372875482222,
      "time_limit_failed_share": 0,
      "top_ece": 0.0985229333333334
     }
    },
    "score": 85.84286362718517
   },
   "capability": 54.99411342282565,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 925,135 input + 1,624 output tokens over 1,624 decisions = 570.664408867 tokens/decision. GPU-time cost USD 0.00560752324780137/1,000; old list-price cost USD 0.0227865763546798/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00560752324780137/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005607523247801365
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "local-jev Qwen3.5-4B",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 26.448995848149686,
    "I_sealed": 21.841730588782582,
    "base": 24.145363218466134,
    "excess": 2.03387761692886,
    "gap": 4.607265259367104,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 52.23659737153346,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 41.49103491664046,
       "judge": 60.44825313117996,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": -56.52173913043479,
       "judge": -30.000000000000004,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 27.1103901729156,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.470606060606066,
       "hard": 15.197777777777777,
       "judge": 40.368135593220345,
       "standard": 17.36888888888889
      }
     },
     "sealed|choice": {
      "cc": 46.955426808141326,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 61.4219200354741,
       "hard": 42.477375565610856,
       "judge": 45.99627560521416,
       "standard": 50.1170094839266
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 12.00000000000001,
       "hard": -70.87378640776699,
       "judge": -18.42105263157895,
       "standard": -15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 18.56976495820642,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 56.15659574468085,
       "hard": 12.02813725490197,
       "judge": 13.700838323353281,
       "standard": 20.162994923857813
      }
     }
    }
   },
   "jevbench_score": 12.014647872457989,
   "key": "localjev-qwen3.5-4b",
   "last_measured_on": "2026-10-02",
   "licence": "MIT code; Apache-2.0 Qwen weights",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9924242424242424,
    "decisive_rate": 0.352,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.84286362718517,
     "cost": 59.269627658320445,
     "intelligence": 24.145363218466134,
     "speed": 86.10068849774444
    },
    "board_release": "v1.6.43",
    "capability": 54.99411342282565,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 83,
    "capability_rank": 109,
    "composite_ci95": {
     "A": [
      6.925385954427524,
      15.24148889625826
     ],
     "B": [
      5.568022650808403,
      12.90281361284615
     ],
     "C": [
      4.8092958016857805,
      10.584367289068235
     ]
    },
    "jevbench_score": 11.438161297684031,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 127,
    "rank": 166,
    "ranks": {
     "A": 166,
     "B": 168,
     "C": 167,
     "capability": 164
    },
    "scores": {
     "A": 11.438161297684031,
     "B": 9.482181005106208,
     "C": 7.943167567836132
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.39534114915877583,
     "p50_s_raw": 0.1226705745793879,
     "p95_s_adjusted": 0.6208106089383364,
     "p95_s_raw": 0.2354053044691682
    },
    "views": {
     "Balanced 33:33:33": 10.008247462291514,
     "Emphasis on Accuracy 60:20:20": 7.6341780052667865,
     "Emphasis on Cost 20:20:60": 11.24977392613748,
     "Emphasis on Speed 20:60:20": 12.519995559254454,
     "Intelligence only": 5.630684842640914
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 213,
   "ranked": true,
   "ranks": {
    "A": 213,
    "B": 213,
    "C": 213,
    "capability": 198
   },
   "repo": "https://github.com/amithgc/local-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 12.014647872457989,
    "B": 9.793835183462825,
    "C": 8.343505466984713
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.42474671369418504,
    "p50_s_raw": 0.1373733568470925,
    "p95_s_adjusted": 0.6419355286285281,
    "p95_s_raw": 0.24596776431426406
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B text model, zero-shot next-token letter probabilities",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.50985112092962,
      86.15050973358518
     ],
     "I": [
      19.830618866047317,
      27.13596492581447
     ],
     "capability": [
      50.794515955411754,
      55.552509505023664
     ],
     "composite": null,
     "seed": 62
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.84286362718517,
     "capability": 54.99411342282565,
     "intelligence": 24.145363218466134
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6026666666666667,
      "calibration": 92.17125,
      "cc": 49.186741566647385,
      "cc_by_type": {
       "choice": 49.186741566647385
      },
      "cc_tiered": 49.596012089837394,
      "cc_tiered_open": 52.23659737153346,
      "cc_tiered_sealed": 46.955426808141326,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.186741566647385
     },
     "noul": {
      "accuracy": 0.34933333333333333,
      "calibration": 81.92005333333331,
      "cc": -30.133333333333333,
      "cc_by_type": {
       "noul": -30.133333333333333
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.45066666666666666,
      "calibration": 83.4372875482222,
      "cc": 22.259017543859695,
      "cc_by_type": {
       "score": 22.259017543859695
      },
      "cc_tiered": 22.84007756556101,
      "cc_tiered_open": 27.1103901729156,
      "cc_tiered_sealed": 18.56976495820642,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 22.259017543859695
     }
    },
    "raw_on_A": {
     "calibration": 81.40355743693019,
     "capability": 54.83642941083503,
     "intelligence": 28.26930138473988
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 10.601711906555591,
    "Emphasis on Accuracy 60:20:20": 7.834906022690478,
    "Emphasis on Cost 20:20:60": 12.703796344031275,
    "Emphasis on Speed 20:60:20": 13.050981286588286,
    "Intelligence only": 5.630684842640914
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "FrontiersMind (FrontiersMind on Hugging Face)",
   "axes": {
    "calibration": 89.11821147102742,
    "cost": 54.82196510185538,
    "intelligence": 55.45890006642501,
    "speed": 85.46346745302206
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04496852459016423,
      "ece_score": 91.00629508196715,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 91.00629508196715,
      "score_before_time_limit": 91.00629508196715,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.26809651474530827,
      "brier": 0.105628298230563,
      "ece": 0.07878981233243955,
      "n": 373,
      "score": 84.24203753351209,
      "score_before_time_limit": 84.24203753351209,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9543010752688172,
      "interval90_width": 0.31496415770609315,
      "n": 372,
      "nrps": 0.07062718985439076,
      "score": 92.10630179760304,
      "score_before_time_limit": 92.10630179760304,
      "time_limit_failed_share": 0,
      "top_ece": 0.043623387096774276
     }
    },
    "score": 89.11821147102742
   },
   "capability": 72.28855576872621,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 763,765 input + 162,504 output tokens over 1,485 decisions = 623.75016835 tokens/decision. GPU-time cost USD 0.0320578576853667/1,000; old list-price cost USD 0.544305252525253/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0320578576853667/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.5388622000000001
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03205785768536673
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "FrontiersMind Lumma-fev-27B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB, driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h, 300 GB disk, host RAM 90,484 MiB. The author's own `python3 -m lumma_fev.server` on 127.0.0.1:8000 inside a docker container with network none (cut proven from inside before any item hash or model use: loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, 16 GB shared memory, read-only weights and code, only /output writable, empty credential variables (names-only check), HF offline flags present in the container configuration (receipt asserted before the cut proof), one request at a time, timed over loopback on the VM. Measured 22 Oct 2026 (VM created 10:31Z; sealed run 10:36:58Z to 10:41:32Z, scored loop 234 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked on the node and inside the container). One pass: 1,500 requests plus a 50-item determinism repeat and a health check. The input was shredded from the node and its absence verified; the VM and its volume were removed by exact id (confirmed 10:42:01Z). Estimated rental cost USD 0.49 for this VM plus USD 0.34 for the earlier first VM that failed before any sealed byte. A first VM (10:19Z to 10:27Z) stopped at the offline-flags proof before any sealed byte was sent and was deleted (see code_revision).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 61.03467151530754,
    "I_sealed": 49.88312861754248,
    "base": 55.45890006642501,
    "excess": 1.3120161274093434,
    "gap": 11.151542897765054,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 78.56754551019564,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 79.2387543252595,
       "judge": 80.22412656558998,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 42.275362318840585,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 62.26110671688638,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 65.7,
       "hard": 60.33759259259259,
       "judge": 70.05949152542374,
       "standard": 52.69111111111111
      }
     },
     "sealed|choice": {
      "cc": 73.84071346202205,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 69.81612119810339,
       "judge": 72.67384916748287,
       "standard": 74.3724952541658
      }
     },
     "sealed|noul": {
      "cc": 27.520998815903607,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 55.172413793103445,
       "hard": 29.0909090909091,
       "judge": 7.692307692307687,
       "standard": 40.29850746268657
      }
     },
     "sealed|score": {
      "cc": 48.2876735747018,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 71.34868421052632,
       "hard": 40.68315492957747,
       "judge": 47.96874999999999,
       "standard": 52.44459090909089
      }
     }
    }
   },
   "jevbench_score": 67.57419732927984,
   "key": "lumma-fev-27b",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (model card metadata, full LICENSE and NOTICE files; serving package lumma-fev 0.2.0 is Apache-2.0 AND MIT; base Qwen/Qwen3.8-27B is Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "FrontiersMind/Lumma-fev-27B@2c080fb4b44bbad2a140353393d935e8b7af8bcb",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9267399267399268,
    "decisive": 273,
    "decisive_rate": 0.7319034852546917,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 99,
   "ranked": true,
   "ranks": {
    "A": 99,
    "B": 95,
    "C": 89,
    "capability": 80
   },
   "repo": "https://huggingface.co/FrontiersMind/Lumma-fev-27B",
   "scores": {
    "A": 67.57419732927984,
    "B": 64.74540060316224,
    "C": 57.732571034098804
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.43387249599995814,
    "p50_s_raw": 0.14193624799997906,
    "p95_s_adjusted": 0.6550751642999671,
    "p95_s_raw": 0.25253758214998356
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The author's own serving package, unmodified: PyPI `lumma-fev` 0.2.0 (wheel sha256 ac8e5f9380b3ea3591ae7b2eb075b2ca28d2d362038267548636e91352b19a91 = the hash PyPI publishes; sdist sha256 60a9b79cd91d7e3fbdaf18c9a16573e1273dd9959eb2c1639e9c6a95401e8c87; five Python files, 445 lines, byte-identical between wheel and sdist; uploaded 14 Oct 2026). The Hub repository itself ships no server code (the AutoJev source folder was removed by the authors on 14 Oct). No shim, no status-mapping proxy and no prompt, answer or usage rewrite was used: the author's server speaks the frozen POST /v1/systemone wire and returns usage natively, the refusals are its own HTTP 422, and the frozen TypeSafeAdapter, our first-party pod runner and pod driver were unchanged. Installed on the rented pod only, from local files, `pip install --no-deps --no-index --force-reinstall` of exactly five hash-checked wheels (lumma_fev-0.2.0, transformers-5.17.0 sha256 78ec1ce21579b38dfb83950a0658cd119f87212a2fcfdff478096ce9d6c03801, huggingface_hub-1.33.0 04e434b06e100eddbce9a6e817d72693a7884b10a79bd67ab48080d5c07eb899, fastapi-0.136.3 3d2a69bdf04b7e9f3afa292c3bc7a98816bbfafa10bc9b45f3f3700d2f761620, uvicorn-0.54.0 505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf) inside a `docker build --network none --pull=false`; nothing is pulled from any index. Base image: stock vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90, used only as a CUDA / PyTorch base (torch 2.13.0+cu130, Python 3.12.3, pydantic 2.13.5, httpx 0.28.1, safetensors 0.8.0, tokenizers 0.23.2); vLLM itself is NOT used, the author serves with plain transformers and PyTorch. Derived image sha256:79d8477ca43c484d5f13bd15d85b44b5c4acd3df7b72281e07ab37d794ec62b6. In the container, before any weights, every installed lumma_fev/*.py was compared to the pinned PyPI file, the exact versions above were asserted, and flash-linear-attention, causal-conv1d and the Hub `kernels` package were asserted NOT importable. Started as `python3 -m lumma_fev.server --model /models/lumma-fev-27b/snapshots/2c080fb4b44bbad2a140353393d935e8b7af8bcb --host 127.0.0.1 --port 8000 --name lumma-fev-27b` with HF_HUB_OFFLINE=1, TRANSFORMERS_OFFLINE=1, USE_HUB_KERNELS=NO and an empty LUMMA_FEV_API_KEY (no --cors, no --api-key, no --device-map). As shipped: bfloat16, sdpa attention, readout temperature 0.93705 (decision_config.json), context limit 8,192 tokens (readout.py:11, not configurable from the command line), one request at a time (the server holds a lock). The optional speed kernels (flash-linear-attention, causal-conv1d) were NOT installed, so transformers' slower PyTorch fallback ran for the gated-delta-net layers; they are unreviewed extra code and a kernel variant would be a separate run. Code gate: cleared by the benchmark owner (22 Oct 2026 10:11Z) on a written list of the disclosed code patterns below. DISCLOSED CODE PATTERNS, none reachable in our configuration: (1) Hub downloads of decision_config.json, readout.safetensors or an image processor (readout.py:22-24, 129-131, 154-155) fire only when --model is not a local folder or a request carries images; we pass a local hash-verified folder and the frozen adapter sends no images; (2) `trust_remote_code=True` (local.py:43) in a fallback reached only when decision_config.json is absent; it is present (sha256 e212eb8cf3eb3759c703a19e0b565e4e06d55e4be96caf96d14b0d3c8de30427, asserted in the container before the cut proof) and the snapshot has no .py file and no auto_map; (3) transformers' optional Hub-kernel loaders in the Qwen3.5 model code (modeling_qwen3_5.py:217, 249-300, 437, 500, 840), inactive because USE_HUB_KERNELS=NO and the `kernels` package is not installed; (4) one environment read (LUMMA_FEV_API_KEY, left empty) and an imported but never used httpx client class (client.py). The authors' code has no subprocess, socket, eval, exec, pickle or torch.load call and no request-body logging. The source review (PASS) and the independent runner review both found nothing that required an owner gate; we asked for the owner's clearance anyway under our foreign-code rule. Reviews: a static source review (PASS), an independent runner review and a delta review (one must: a receipt proving the offline flags inside the scored container before the cut proof, closed and re-checked by us before launch); the sealed-input handling and the rented device were admitted with conditions, and the conditions were met. A first VM (before any sealed byte existed) failed exactly that offline-flags proof, because the flags reached the scored container only through its `env -i` entrypoint and not through the Docker container configuration; we then added the same three flags as `-e` options on `docker run` for the scored container (a one-line runner change checked by us with the static checks and the offline proof, fit and guard tests, not by a further independent review round) and the measurement ran once on a second VM; the first VM was deleted. Weights: FrontiersMind/Lumma-fev-27B at 2c080fb4b44bbad2a140353393d935e8b7af8bcb (weights, config, decision_config.json, readout.safetensors, tokenizer and chat template unchanged since the first upload on 14 Oct 2026; later commits only README, NOTICE and benchmark images), 23 files (11 bfloat16 shards, readout.safetensors, tokenizer.json, 10 small files; 52,193,544,229 bytes; manifest sha256 8411f98f33617fea738d6b6ecb932f6c45bbd214391d3615d4043a5e7d09d590), fetched by us from the commit-pinned Hub revision on the pod before the network was cut, and sha256-verified file by file (shard sha256 = the Hub LFS oid, small files = git blob oid; 'VERIFY OK 23 files') on the host before the smoke; the three benchmark images were not used. FIRST MEASUREMENT (found by our own intake: Decision Index 0.4.1 entry; no outside request for this model and no earlier JevBench measurement). AUTHOR-DEFAULT CONTEXT LIMIT: the author's server accepts at most 8,192 tokens per question including state, options and chat template and never truncates; above that it answers HTTP 422 itself ('a question needs N tokens with its state; the limit is 8192 and input is never truncated'). The 15 items of 76,368-80,526 tokens were refused exactly as pre-registered by id and position and are scored wrong, like the other self-hosted rows on this draw; the largest answered item is 1,864 tokens, nowhere near the limit. No status proxy was used. SERVER IS THE AUTHOR'S OWN: lumma_fev.server from the PyPI package, not the Hub repository (which ships no server code); see code_revision for the exact bytes, the disclosed code patterns and the reviews. SCORE QUESTIONS ARE OUTSIDE THE TRAINED TYPES: the card states the model was trained on noul and choice questions and that score questions are accepted but untested; about a quarter of this draw's items are score questions, answered natively by the same readout (probabilities per level), with nothing added or rewritten by us. OPTIONAL KERNELS OFF: flash-linear-attention and causal-conv1d (recommended by the card for speed) were not installed, so transformers' slower PyTorch fallback ran; kernel variants can change speed and numerical results; numerical parity against an optional-kernel configuration was not tested. SERIAL ONLY: the server handles one request at a time and our driver sent one at a time; determinism over the first 50 positions was bit-identical (50/50, 0 argmax flips, max probability difference 0.0). TRAINING DATA: the card gives only 'about 100K samples' in English, Hindi, Tamil and Kannada and names no datasets; there is no training-data licence table and no contamination or overlap statement in the pinned card (the authors' blog post linked from the card was not read). The Decision Index 0.4.1 row carries no overlap or contamination flag. We ran no overlap check, and no JevBench use or denial is declared, so whether its training data overlaps JevBench sources is unknown to us. LICENCE: Apache-2.0 in the card metadata with a full LICENSE and a NOTICE file; the serving package is Apache-2.0 AND MIT (its prompt and scoring follow AutoJev, MIT). SELF-REPORTED, unverified: the card's three benchmark charts (accuracy against 'TypeSafe Jev 1.13.0' on public sets; not transcribed or checked here) and no JevBench or Decision Index number is claimed by the authors. The Decision Index 0.4.1 (22 Oct 2026 snapshot) states for this model: full (balanced_skill) score 59.97, public 57.15 (its kit rescore equals the submitted claim), same-skills 56.59, new-domains 51.01, decision agreement with the submitted rows 99.25% on 825 rows against a 98.5% bar (private tests run by the lab on one RTX PRO 6000 with the author's serving), latency median 100 ms and p95 1,075 ms, and that questions over its declared 8,192-token limit count as unsupported. 10 s RULE: max measured decision latency 0.48 s over all 1,500 rows (answered items: median 0.14 s, p95 0.25 s); 0 decisions over 10 s. In the synthetic smoke before the run a 8,000-token request took 2.17 s and an exact 8,192-token request 2.22 s, but no answered item is larger than 1,864 tokens. OUTPUT TOKENS: our pre-registered output-token planning band was missed (see cost_basis); every other pre-registered prediction (refusal ids and positions, answered count, input tokens item by item, determinism) was met.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 70.49,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 36.47
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 70.49
      },
      "calibration_gen": {
       "accuracy": 0.6039603960396039,
       "cc": 37.87697574231177,
       "cc_by_type": {
        "choice": 65.00530222693531,
        "noul": 4.0000000000000036,
        "score": 44.625625
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 37.87697574231177
      },
      "composite_gen": {
       "accuracy": 0.801980198019802,
       "cc": 58.0487336993028,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 25.92592592592593,
        "score": 51.47230769230768
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 58.0487336993028
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.50870370370372,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 90.52611111111112
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.50870370370372
      },
      "jaggedness": {
       "accuracy": 0.5277777777777778,
       "cc": 18.788328277890084,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": -29.729729729729726,
        "score": 43.84254901960786
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 18.788328277890084
      },
      "llm_router": {
       "accuracy": 0.8945686900958466,
       "cc": 79.20319390999602,
       "cc_by_type": {
        "choice": 90.49044642070969,
        "noul": 85.56701030927836,
        "score": 61.55212499999998
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 79.20319390999602
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -14.169947635411548,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -25,
        "score": -30.91190476190475
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.7127272727272728,
       "cc": 51.028387885306735,
       "cc_by_type": {
        "choice": 73.58982719269645,
        "noul": 19.444444444444443,
        "score": 60.05089201877934
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 51.028387885306735
      },
      "negation_pair_gen": {
       "accuracy": 0.7441860465116279,
       "cc": 42.64922222222223,
       "cc_by_type": {
        "choice": 100,
        "noul": 6.666666666666665,
        "score": 21.281000000000017
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 42.64922222222223
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 76.1606853006853,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 80,
        "score": 69.50307692307692
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 76.1606853006853
      },
      "usecase": {
       "accuracy": 0.7272727272727273,
       "cc": 54.57329856209151,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 37.5,
        "score": 53.180680000000024
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 54.57329856209151
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 1.5839194139193964,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": -88.10538461538466
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 1.5839194139193964
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 35.98355555555555,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 41.284
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 35.98355555555555
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 76.09684750545081,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 77.18434782608696
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 76.09684750545081
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.48848230912478,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 99.98500000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.48848230912478
      },
      "en": {
       "accuracy": 0.7523892267593397,
       "cc": 55.34845437949974,
       "cc_by_type": {
        "choice": 76.27561294101854,
        "noul": 36.690647482014384,
        "score": 53.0791027154663
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 55.34845437949974
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 63.926572945654605,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 1.5676470588235292
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 63.926572945654605
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.65326047358833,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 99.41333333333334
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.65326047358833
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 82.40638802889576,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 89.57210526315788
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 82.40638802889576
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 33.07397346368713,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 25,
        "score": 24.501249999999942
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 33.07397346368713
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 61.90822254388893,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -0.24571428571433795
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 61.90822254388893
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 26.341737004677427,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 0,
        "score": 7.619333333333311
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 26.341737004677427
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 1.1280423280423264,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 47.82857142857143
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 1.1280423280423264
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.97777777777777,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.93333333333332
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.97777777777777
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -0.30134199134198525,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 25.070000000000014
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 51.123878101349284,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 0,
        "score": 60.98906976744185
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 51.123878101349284
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 69.44292705026581,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": 61.36354838709678
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 69.44292705026581
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 45.32205673758866,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 19.999999999999996,
        "score": 67.03000000000002
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 45.32205673758866
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 44.2522463768116,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50,
        "score": 26.23500000000002
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 44.2522463768116
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 76.56564516129032,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 61.955000000000005
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 76.56564516129032
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 30.91888888888889,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 26.090000000000014
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 30.91888888888889
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8093333333333333,
       "cc": 75.37266308195717,
       "cc_by_type": {
        "choice": 75.37266308195717
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 75.37266308195717
      },
      "noul": {
       "accuracy": 0.6746666666666666,
       "cc": 34.93333333333333,
       "cc_by_type": {
        "noul": 34.93333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 34.93333333333333
      },
      "score": {
       "accuracy": 0.672,
       "cc": 51.95379963898918,
       "cc_by_type": {
        "score": 51.95379963898918
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 51.95379963898918
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -7.332352941176471,
       "cc_by_type": {
        "choice": 52,
        "noul": -75,
        "score": 1.0029411764705842
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.7083333333333334,
       "cc": 51.67661411052716,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 100,
        "score": -7.703076923076924
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 51.67661411052716
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 44.13327913279132,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50,
        "score": -1.3400000000000079
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 44.13327913279132
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -19.73008040935672,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -33.333333333333336,
        "score": -20.59374999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 34.75663647342995,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 66.58874999999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 34.75663647342995
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -33.22037735849057,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": -182.68
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.7,
       "cc": 58.02814264167205,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 50.835128205128186
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 58.02814264167205
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 26.271241226076857,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -50,
        "score": 60.17260869565217
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 26.271241226076857
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 4.0666666666666655,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60,
        "score": 55.533333333333324
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 4.0666666666666655
      },
      "knowledge_graphs": {
       "accuracy": 0.6818181818181818,
       "cc": 50.038830921636055,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 33.33333333333333,
        "score": 42.4790909090909
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 50.038830921636055
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 50.424990603711535,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 29.569545454545477
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 50.424990603711535
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 10.750358422939065,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 27.949999999999996
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 10.750358422939065
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 64.90648148148148,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 74.71944444444443
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 64.90648148148148
      },
      "model_routing": {
       "accuracy": 0.8894009216589862,
       "cc": 77.00875755321067,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 87.87878787878789,
        "score": 48.99164062499999
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 77.00875755321067
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 19.810444093184298,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 70.94764705882352
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 19.810444093184298
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 57.81635752208175,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 73.5419230769231
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.81635752208175
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 35.491342606007926,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": 21.25235294117648
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 35.491342606007926
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 19.471690821256033,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 35.22666666666665
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 19.471690821256033
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 97.71456140350877,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 93.14368421052632
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.71456140350877
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.10373154216826,
      89.78560246866702
     ],
     "I": [
      50.35029169471709,
      59.53366527085479
     ],
     "capability": [
      68.38797728474432,
      73.87563540528951
     ],
     "composite": null,
     "seed": 52
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.11821147102742,
     "capability": 72.28855576872621,
     "intelligence": 55.45890006642501
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8093333333333333,
      "calibration": 91.00629508196715,
      "cc": 75.37266308195717,
      "cc_by_type": {
       "choice": 75.37266308195717
      },
      "cc_tiered": 76.20412948610885,
      "cc_tiered_open": 78.56754551019564,
      "cc_tiered_sealed": 73.84071346202205,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.37266308195717
     },
     "noul": {
      "accuracy": 0.6746666666666666,
      "calibration": 84.24203753351209,
      "cc": 34.93333333333333,
      "cc_by_type": {
       "noul": 34.93333333333333
      },
      "cc_tiered": 34.898180567372094,
      "cc_tiered_open": 42.275362318840585,
      "cc_tiered_sealed": 27.520998815903607,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 34.93333333333333
     },
     "score": {
      "accuracy": 0.672,
      "calibration": 92.10630179760304,
      "cc": 51.95379963898918,
      "cc_by_type": {
       "score": 51.95379963898918
      },
      "cc_tiered": 55.27439014579409,
      "cc_tiered_open": 62.26110671688638,
      "cc_tiered_sealed": 48.2876735747018,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 51.95379963898918
     }
    },
    "raw_on_A": {
     "calibration": 85.46331356109388,
     "capability": 70.8690106210405,
     "intelligence": 56.274707680987134
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 62.534992507236694,
    "Emphasis on Accuracy 60:20:20": 59.49839329224038,
    "Emphasis on Cost 20:20:60": 59.2032181484216,
    "Emphasis on Speed 20:60:20": 70.05258571689802,
    "Intelligence only": 55.45890006642501
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "LycheeAI",
   "axes": {
    "calibration": 85.36006033410777,
    "cost": 57.33528640641721,
    "intelligence": 74.33387897478502,
    "speed": 83.81641932679139
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08797487946564417,
      "ece_score": 82.40502410687117,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 81.30629045211289,
      "score_before_time_limit": 82.40502410687117,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.08266666666666667,
      "brier": 0.07113655151967689,
      "ece": 0.059544040488169664,
      "n": 375,
      "score": 87.62137221222012,
      "score_before_time_limit": 88.09119190236608,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.936,
      "interval90_width": 0.14511111111111116,
      "n": 375,
      "nrps": 0.04892728578465192,
      "score": 87.15251833799032,
      "score_before_time_limit": 87.85536122781282,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.09698274482954589
     }
    },
    "score": 85.36006033410777
   },
   "capability": 79.8469696544464,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 763,765 input + 0 output tokens over 1,485 decisions = 514.31986532 tokens/decision. GPU-time cost USD 0.026433649053422/1,000; old list-price cost USD 0.216014343434343/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.026433649053422/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.5442374
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02643364905342198
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "Measured on 13 Oct 2026 as a paid priority evaluation; published at the customer's request. Over the cost cap under the earlier list-price basis; inside both caps from v1.8.0 (GPU-time cost basis).",
   "display": "Lychee-27B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.20",
    "results_sha256": "b55bb2e6bc838bdf70a0d0fb92581f422d2ca9a1bbba7726c43d041369387568",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (A100 80GB PCIe), offline read-only container (--network none); author's lychee server unchanged at revision f3f95d18 (bf16, one forward pass, linear readout over answer codes, temperature 1.0, prompts read whole; over-length/OOM -> HTTP 422) on the documented stack Python 3.12 / torch 2.14.0 (cu126) / transformers 5.17.0 / flash-linear-attention 0.5.2 / kernels 0.16.1",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 77.65480496064171,
    "I_sealed": 71.01295298892833,
    "base": 74.33387897478502,
    "excess": -3.197674798642332,
    "gap": 6.641851971713379,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 82.15362883943581,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 81.12614029569046,
       "judge": 92.089650626236,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 83.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 70,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 67.73349135649897,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 79.81615491246663,
       "hard": 75.21139265816639,
       "judge": 77.90948504830118,
       "standard": 31.47236643747696
      }
     },
     "sealed|choice": {
      "cc": 79.93815932929519,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 76.06106163987509,
       "judge": 77.08129285014691,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 65.36948486408085,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 58.18181818181818,
       "judge": 60.00000000000001,
       "standard": 79.10447761194031
      }
     },
     "sealed|score": {
      "cc": 67.73121477340896,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 90.27477600247926,
       "hard": 62.8838167634306,
       "judge": 66.28447449122841,
       "standard": 68.32434060210133
      }
     }
    }
   },
   "jevbench_score": 73.34066782166333,
   "key": "lychee-27b",
   "last_measured_on": "2026-10-13",
   "licence": "CC BY-NC 4.0 (weights; non-commercial use only); base Qwen3.8-27B Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.7.20",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": {
    "base": "Qwen/Qwen3.8-27B",
    "model": "LycheeAI/Lychee-27B",
    "revision": "f3f95d18a3b1ef3c483c2a5d7e8e5272a2af31e2"
   },
   "new_in": "v1.7.20",
   "noul_decisive": {
    "acc_among_decisive": 0.9385964912280702,
    "decisive": 342,
    "decisive_rate": 0.9168900804289544,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "provider": "self-hosted",
   "rank": 43,
   "ranked": true,
   "ranks": {
    "A": 43,
    "B": 23,
    "C": 36,
    "capability": 34
   },
   "repo": "https://huggingface.co/LycheeAI/Lychee-27B",
   "scores": {
    "A": 73.34066782166333,
    "B": 73.53718103847346,
    "C": 73.34066782166333
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.504129411996837,
    "p50_s_raw": 0.1770647059984185,
    "p95_s_adjusted": 0.823789081400173,
    "p95_s_raw": 0.3368945407000865
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 96.84494590858434,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 98.8681710590863
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 96.84494590858434
      },
      "calibration_gen": {
       "accuracy": 0.7425742574257426,
       "cc": 58.02702947266487,
       "cc_by_type": {
        "choice": 61.82396606574761,
        "noul": 43.99999999999999,
        "score": 68.257122352247
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 58.02702947266487
      },
      "composite_gen": {
       "accuracy": 0.9108910891089109,
       "cc": 88.37566567915849,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 79.03846948553324
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 88.37566567915849
      },
      "fairness": {
       "accuracy": 0.9583333333333334,
       "cc": 91.06863217886082,
       "cc_by_type": {
        "choice": 100,
        "noul": 73.33333333333334,
        "score": 99.87256320324913
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 91.06863217886082
      },
      "jaggedness": {
       "accuracy": 0.7,
       "cc": 55.99945684554999,
       "cc_by_type": {
        "choice": 59.57651588065449,
        "noul": 45.945945945945944,
        "score": 62.47590871004955
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 55.99945684554999
      },
      "llm_router": {
       "accuracy": 0.9424920127795527,
       "cc": 87.40131830395467,
       "cc_by_type": {
        "choice": 96.83014880690322,
        "noul": 100,
        "score": 65.37380610496079
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 87.40131830395467
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7037037037037037,
       "cc": 45.91549422363414,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 0,
        "score": 87.23101875337665
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 45.91549422363414
      },
      "multilingual": {
       "accuracy": 0.7963636363636364,
       "cc": 68.48993580117428,
       "cc_by_type": {
        "choice": 77.50244538637104,
        "noul": 61.111111111111114,
        "score": 66.8562509060407
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 68.48993580117428
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 82.07310706341941,
       "cc_by_type": {
        "choice": 100,
        "noul": 73.33333333333334,
        "score": 72.88598785692493
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 82.07310706341941
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 85.12101684349959,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 70.37806554551376
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 85.12101684349959
      },
      "usecase": {
       "accuracy": 0.7863636363636364,
       "cc": 69.10147568968794,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 58.33333333333333,
        "score": 72.25540746122066
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 69.10147568968794
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 70.32824435845613,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100,
        "score": 68.12759021822555
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 70.32824435845613
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 54.17277223403257,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 29.184983368764406
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 54.17277223403257
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 87.52763656022877,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 78.51211322050935
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.52763656022877
      },
      "el": {
       "accuracy": 0.8666666666666667,
       "cc": 83.3018794083968,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 66.66666666666667,
        "score": 99.9987480948366
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 83.3018794083968
      },
      "en": {
       "accuracy": 0.8427454387489139,
       "cc": 75.65866148884379,
       "cc_by_type": {
        "choice": 82.2067097057639,
        "noul": 74.10071942446044,
        "score": 70.66855533630705
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 75.65866148884379
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 76.60509808096833,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 39.6032224647647
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 76.60509808096833
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.84364531764771,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 99.98448786551147
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.84364531764771
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 85.36100932828727,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 98.43596916133238
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 85.36100932828727
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 67.38707139277147,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 25,
        "score": 77.16121417831445
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 67.38707139277147
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8,
       "cc": 57.932625787928835,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -12.17250455359462
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 57.932625787928835
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 39.24346883797747,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 19.999999999999996,
        "score": 12.027467673582926
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 39.24346883797747
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 21.44829518164472,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 64.34488554493416
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 21.44829518164472
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.9979027190055,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.99370815701651
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.9979027190055
      },
      "nl": {
       "accuracy": 0.6,
       "cc": -8.469841369789243,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": -66.10216480200843
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 78.24686689600323,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80,
        "score": 69.97547161479767
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 78.24686689600323
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 78.50805243593841,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 67.34501764959485
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 78.50805243593841
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 41.46060672840398,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 60.00000000000001,
        "score": 40.97756486606301
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 41.46060672840398
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 70.3760867224964,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 61.12826016748918
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 70.3760867224964
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 67.72540002497259,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": 100,
        "score": 99.95039362330482
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.72540002497259
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 67.72822990777657,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 69.8513563899964
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 67.72822990777657
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8546666666666667,
       "cc": 81.22811381771561,
       "cc_by_type": {
        "choice": 81.22811381771561
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.22811381771561
      },
      "noul": {
       "accuracy": 0.856,
       "cc": 71.2,
       "cc_by_type": {
        "noul": 71.2
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 71.2
      },
      "score": {
       "accuracy": 0.7653333333333333,
       "cc": 69.11212137268383,
       "cc_by_type": {
        "score": 69.11212137268383
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 69.11212137268383
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 26.31153461221812,
       "cc_by_type": {
        "choice": 68,
        "noul": -50,
        "score": 60.93460383665437
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 26.31153461221812
      },
      "code_linting": {
       "accuracy": 0.9583333333333334,
       "cc": 90.54762535995049,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 90.27641645252226
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 90.54762535995049
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 61.80541818609667,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 17.93657976154204
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 61.80541818609667
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -22.944869063561228,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": -108.6006890620287
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 31.53974994106892,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 71.4308440261053
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 31.53974994106892
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": -27.169086458497507,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": -202.2619763566246
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.8666666666666667,
       "cc": 71.8270180298331,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 42.85714285714286,
        "score": 72.62391123235643
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 71.8270180298331
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 76.29037483638065,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50,
        "score": 89.32408618161584
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 76.29037483638065
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 25.653502774317133,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60,
        "score": 78.62717498961807
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 25.653502774317133
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 74.12884070837195,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100,
        "score": 60.93041934138995
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 74.12884070837195
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 71.44881097236704,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 48.196562116067554
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 71.44881097236704
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 29.824646023222176,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 52.9147982847203
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 29.824646023222176
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 89.8093978309928,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 69.42819349297838
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 89.8093978309928
      },
      "model_routing": {
       "accuracy": 0.9170506912442397,
       "cc": 83.7827340206028,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 100,
        "score": 56.02352673713311
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 83.7827340206028
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 49.26720682395484,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 0,
        "score": 82.35056480967641
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 49.26720682395484
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 81.75864388980165,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 100,
        "score": 78.70211551341609
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 81.75864388980165
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 29.49924629216827,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": 18.05438912281022
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 29.49924629216827
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 47.61801645268476,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 67.49173051747456
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 47.61801645268476
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.99892144993377,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.99676434980128
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.99892144993377
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.30822099712003,
      87.15491104595135
     ],
     "I": [
      68.70295566404981,
      78.08685181186851
     ],
     "capability": [
      75.75191839613736,
      82.09717042413354
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.36006033410777,
     "capability": 79.8469696544464,
     "intelligence": 74.33387897478502
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8546666666666667,
      "calibration": 81.30629045211289,
      "cc": 81.22811381771561,
      "cc_by_type": {
       "choice": 81.22811381771561
      },
      "cc_tiered": 81.0458940843655,
      "cc_tiered_open": 82.15362883943581,
      "cc_tiered_sealed": 79.93815932929519,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.22811381771561
     },
     "noul": {
      "accuracy": 0.856,
      "calibration": 87.62137221222012,
      "cc": 71.2,
      "cc_by_type": {
       "noul": 71.2
      },
      "cc_tiered": 74.2233897750356,
      "cc_tiered_open": 83.07729468599035,
      "cc_tiered_sealed": 65.36948486408085,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 71.2
     },
     "score": {
      "accuracy": 0.7653333333333333,
      "calibration": 87.15251833799032,
      "cc": 69.11212137268383,
      "cc_by_type": {
       "score": 69.11212137268383
      },
      "cc_tiered": 67.73235306495397,
      "cc_tiered_open": 67.73349135649897,
      "cc_tiered_sealed": 67.73121477340896,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 69.11212137268383
     }
    },
    "raw_on_A": {
     "calibration": 82.39849260812656,
     "capability": 77.78229880579507,
     "intelligence": 73.16610500346357
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.05267201429658,
    "Emphasis on Accuracy 60:20:20": 71.70458179714531,
    "Emphasis on Cost 20:20:60": 64.34389364557737,
    "Emphasis on Speed 20:60:20": 74.97759063981064,
    "Intelligence only": 74.33387897478502
   }
  },
  {
   "adapter_id": "images/malkuth-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "newfull5 (dhtocks)",
   "axes": {
    "calibration": 79.34885520197874,
    "cost": 83.82259492095744,
    "intelligence": 15.917395195670291,
    "speed": 91.96529998043799
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1264719869706841,
      "ece_score": 74.70560260586319,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 74.70560260586319,
      "score_before_time_limit": 74.70560260586319,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.46630727762803237,
      "brier": 0.1896760988948787,
      "ece": 0.09166172506738543,
      "n": 371,
      "score": 81.6676549865229,
      "score_before_time_limit": 81.6676549865229,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9024390243902439,
      "interval90_width": 0.6144986449864495,
      "n": 369,
      "nrps": 0.14966934108401095,
      "score": 81.67330801355013,
      "score_before_time_limit": 81.67330801355013,
      "time_limit_failed_share": 0,
      "top_ece": 0.10843224932249321
     }
    },
    "score": 79.34885520197874
   },
   "capability": 47.633125198824516,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 760,783 input + 169,020 output tokens over 1,624 decisions = 572.538793103 tokens/decision. GPU-time cost USD 0.00346136052100438/1,000; old list-price cost USD 0.0140538731527094/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00346136052100438/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0034613605210043795
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Malkuth-2B (newfull5, Kev post-train)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 16.94824627575026,
    "I_sealed": 14.88654411559032,
    "base": 15.917395195670291,
    "excess": -0.511685482278303,
    "gap": 2.061702160159941,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 24.95619850881217,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 16.955017301038072,
       "judge": 28.80685563612393,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": -21.739130434782606,
       "judge": -40,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 25.88854031843862,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.77636363636364,
       "hard": 12.745185185185182,
       "judge": 28.807457627118627,
       "standard": 35.852962962962955
      }
     },
     "sealed|choice": {
      "cc": 39.92521849097588,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 50.77969107974283,
       "hard": 26.18778280542985,
       "judge": 44.134078212290504,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 28.000000000000004,
       "hard": -39.80582524271845,
       "judge": -23.684210526315784,
       "standard": 18.309859154929576
      }
     },
     "sealed|score": {
      "cc": 4.734413855795083,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 40.8195035460993,
       "hard": -11.993921568627375,
       "judge": 11.762814371257434,
       "standard": 9.605939086294368
      }
     }
    }
   },
   "jevbench_score": 4.126837854112484,
   "key": "malkuth-2b",
   "last_measured_on": "2026-10-02",
   "licence": "CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8434343434343434,
    "decisive_rate": 0.5336927223719676,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.34885520197874,
     "cost": 65.5661191189412,
     "intelligence": 15.917395195670291,
     "speed": 92.03462875001455
    },
    "board_release": "v1.6.43",
    "capability": 47.633125198824516,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 96,
    "capability_rank": 123,
    "composite_ci95": {
     "A": [
      2.1896531510798423,
      6.224018858922979
     ],
     "B": [
      1.639660951009494,
      4.9293547200832295
     ],
     "C": [
      1.5205924660276677,
      4.322235318696512
     ]
    },
    "jevbench_score": 3.992169201455145,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 146,
    "rank": 186,
    "ranks": {
     "A": 186,
     "B": 189,
     "C": 187,
     "capability": 182
    },
    "scores": {
     "A": 3.992169201455145,
     "B": 3.08287077876474,
     "C": 2.7723397232327396
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23610859100008383,
     "p50_s_raw": 0.04305429550004192,
     "p95_s_adjusted": 0.26510952737880866,
     "p95_s_raw": 0.05755476368940435
    },
    "views": {
     "Balanced 33:33:33": 3.4183795643165196,
     "Emphasis on Accuracy 60:20:20": 2.361369226233909,
     "Emphasis on Cost 20:20:60": 4.242339522263916,
     "Emphasis on Speed 20:60:20": 4.578614246342653,
     "Intelligence only": 1.6131545908775944
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 228,
   "ranked": true,
   "ranks": {
    "A": 228,
    "B": 230,
    "C": 228,
    "capability": 220
   },
   "repo": "https://github.com/newfull5/malkuth",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 4.126837854112484,
    "B": 3.1462994379047737,
    "C": 2.8658596209114466
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23758355304598808,
    "p50_s_raw": 0.04379177652299404,
    "p95_s_adjusted": 0.26770324098761195,
    "p95_s_raw": 0.05885162049380597
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.7053529716356,
      81.31914491234029
     ],
     "I": [
      12.692506481394203,
      18.87608002604669
     ],
     "capability": [
      43.451731383062054,
      49.395128138793496
     ],
     "composite": null,
     "seed": 63
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.34885520197874,
     "capability": 47.633125198824516,
     "intelligence": 15.917395195670291
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5133333333333333,
      "calibration": 74.70560260586319,
      "cc": 37.76228413364529,
      "cc_by_type": {
       "choice": 37.76228413364529
      },
      "cc_tiered": 32.44070849989403,
      "cc_tiered_open": 24.95619850881217,
      "cc_tiered_sealed": 39.92521849097588,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 37.76228413364529
     },
     "noul": {
      "accuracy": 0.44533333333333336,
      "calibration": 81.6676549865229,
      "cc": -10.933333333333328,
      "cc_by_type": {
       "noul": -10.933333333333328
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.38133333333333336,
      "calibration": 81.67330801355013,
      "cc": 9.127684210526233,
      "cc_by_type": {
       "score": 9.127684210526233
      },
      "cc_tiered": 15.31147708711685,
      "cc_tiered_open": 25.88854031843862,
      "cc_tiered_sealed": 4.734413855795083,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 9.127684210526233
     }
    },
    "raw_on_A": {
     "calibration": 76.45071634453062,
     "capability": 47.2825063700716,
     "intelligence": 18.114296395612577
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 3.5506640056068495,
    "Emphasis on Accuracy 60:20:20": 2.398404580279615,
    "Emphasis on Cost 20:20:60": 4.628155182229364,
    "Emphasis on Speed 20:60:20": 4.71921297156434,
    "Intelligence only": 1.6131545908775944
   }
  },
  {
   "adapter_id": "images/malkuth-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "newfull5 (dhtocks)",
   "axes": {
    "calibration": 88.43741591513763,
    "cost": 77.49131338115862,
    "intelligence": 33.840703805987175,
    "speed": 89.3735021970277
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0565084690553745,
      "ece_score": 88.6983061889251,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.6983061889251,
      "score_before_time_limit": 88.6983061889251,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3504043126684636,
      "brier": 0.13494345407008085,
      "ece": 0.06141401617250676,
      "n": 371,
      "score": 87.71719676549866,
      "score_before_time_limit": 87.71719676549866,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.926829268292683,
      "interval90_width": 0.5194218608852753,
      "n": 369,
      "nrps": 0.1123930174593495,
      "score": 88.89674479098915,
      "score_before_time_limit": 88.89674479098915,
      "time_limit_failed_share": 0,
      "top_ece": 0.054836043360433705
     }
    },
    "score": 88.43741591513763
   },
   "capability": 61.139059860562405,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 760,783 input + 169,222 output tokens over 1,624 decisions = 572.66317734 tokens/decision. GPU-time cost USD 0.00562716375894004/1,000; old list-price cost USD 0.0296839839901478/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00562716375894004/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005627163758940036
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Malkuth-4B (newfull5, Kev post-train)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 36.78555463394936,
    "I_sealed": 30.895852978024994,
    "base": 33.840703805987175,
    "excess": 3.316314013486122,
    "gap": 5.889701655924366,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 57.213904436116636,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 47.153192827933324,
       "judge": 68.35860250494396,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 19.189786059351285,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": -19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 33.95297340638018,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.45848484848485,
       "hard": 32.323055555555534,
       "judge": 29.900169491525418,
       "standard": 34.53925925925927
      }
     },
     "sealed|choice": {
      "cc": 53.38404633359997,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 69.40359175227256,
       "hard": 43.49547511312216,
       "judge": 51.58286778398511,
       "standard": 67.8531838896416
      }
     },
     "sealed|noul": {
      "cc": 12.571668333969063,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48,
       "hard": -16.504854368932044,
       "judge": 26.315789473684205,
       "standard": 32.3943661971831
      }
     },
     "sealed|score": {
      "cc": 26.73184426650596,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.09113475177305,
       "hard": 12.109877450980411,
       "judge": 28.95095808383231,
       "standard": 34.467461928933965
      }
     }
    }
   },
   "jevbench_score": 28.21050157509396,
   "key": "malkuth-4b",
   "last_measured_on": "2026-10-02",
   "licence": "CC-BY-NC-4.0, research use only (XNLI and RACE in the training mix)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.921161825726141,
    "decisive_rate": 0.6495956873315364,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.43741591513763,
     "cost": 55.82433433950372,
     "intelligence": 33.840703805987175,
     "speed": 89.51571180654442
    },
    "board_release": "v1.6.43",
    "capability": 61.139059860562405,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 58,
    "capability_rank": 83,
    "composite_ci95": {
     "A": [
      17.563735957847147,
      34.57907875172623
     ],
     "B": [
      15.042433721702132,
      31.023588122626016
     ],
     "C": [
      12.197038859616075,
      24.013249133143216
     ]
    },
    "jevbench_score": 26.197476126697616,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 76,
    "rank": 109,
    "ranks": {
     "A": 109,
     "B": 112,
     "C": 122,
     "capability": 128
    },
    "scores": {
     "A": 26.197476126697616,
     "B": 23.020714347323178,
     "C": 18.19269175465112
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2985552769736387,
     "p50_s_raw": 0.07427763848681934,
     "p95_s_adjusted": 0.3744588227418717,
     "p95_s_raw": 0.11222941137093585
    },
    "views": {
     "Balanced 33:33:33": 23.437149264983223,
     "Emphasis on Accuracy 60:20:20": 19.45371080344073,
     "Emphasis on Cost 20:20:60": 24.246787312645555,
     "Emphasis on Speed 20:60:20": 28.28433154980761,
     "Intelligence only": 15.501658014110385
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 169,
   "ranked": true,
   "ranks": {
    "A": 169,
    "B": 170,
    "C": 172,
    "capability": 161
   },
   "repo": "https://github.com/newfull5/malkuth",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 28.21050157509396,
    "B": 24.236501993606172,
    "C": 19.590626093815246
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3005055939778686,
    "p50_s_raw": 0.07525279698893428,
    "p95_s_adjusted": 0.3844122505222912,
    "p95_s_raw": 0.11720612526114563
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.33637885013427,
      89.07368375555646
     ],
     "I": [
      28.800746942905594,
      38.02075761880322
     ],
     "capability": [
      56.551766449658025,
      62.63523991792939
     ],
     "composite": null,
     "seed": 64
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.43741591513763,
     "capability": 61.139059860562405,
     "intelligence": 33.840703805987175
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6466666666666666,
      "calibration": 88.6983061889251,
      "cc": 54.813713138126026,
      "cc_by_type": {
       "choice": 54.813713138126026
      },
      "cc_tiered": 55.2989753848583,
      "cc_tiered_open": 57.213904436116636,
      "cc_tiered_sealed": 53.38404633359997,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.813713138126026
     },
     "noul": {
      "accuracy": 0.592,
      "calibration": 87.71719676549866,
      "cc": 18.399999999999995,
      "cc_by_type": {
       "noul": 18.399999999999995
      },
      "cc_tiered": 15.880727196660175,
      "cc_tiered_open": 19.189786059351285,
      "cc_tiered_sealed": 12.571668333969063,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.399999999999995
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 88.89674479098915,
      "cc": 29.251991228070175,
      "cc_by_type": {
       "score": 29.251991228070175
      },
      "cc_tiered": 30.34240883644307,
      "cc_tiered_open": 33.95297340638018,
      "cc_tiered_sealed": 26.73184426650596,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 29.251991228070175
     }
    },
    "raw_on_A": {
     "calibration": 85.35554104202156,
     "capability": 62.08736671102251,
     "intelligence": 38.819192380023466
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 25.617680606591044,
    "Emphasis on Accuracy 60:20:20": 20.31487424510763,
    "Emphasis on Cost 20:20:60": 28.82685221359758,
    "Emphasis on Speed 20:60:20": 30.12798990064605,
    "Intelligence only": 15.501658014110385
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "baselabs (Base Labs at Baseten; baselabs on Hugging Face)",
   "axes": {
    "calibration": 94.26152926942419,
    "cost": 57.166365164587916,
    "intelligence": 76.20447479084162,
    "speed": 84.40895777972253
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.024914839344262127,
      "ece_score": 95.01703213114757,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 95.01703213114757,
      "score_before_time_limit": 95.01703213114757,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.10723860589812328,
      "brier": 0.04332660199162734,
      "ece": 0.027278206434316478,
      "n": 373,
      "score": 94.5443587131367,
      "score_before_time_limit": 94.5443587131367,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9865591397849462,
      "interval90_width": 0.2329749103942652,
      "n": 372,
      "nrps": 0.04045206609657794,
      "score": 93.22319696398831,
      "score_before_time_limit": 93.22319696398831,
      "time_limit_failed_share": 0,
      "top_ece": 0.04754199731182791
     }
    },
    "score": 94.26152926942419
   },
   "capability": 85.2330020301329,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11840.1727884 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=base-arch). The row's own scored cost basis: 777,323 input + 1,485 output tokens over 1,485 decisions = 524.44983165 tokens/decision. GPU-time cost USD 0.0267785975171361/1,000; old list-price cost USD 0.103632717171717/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0267785975171361/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.10259639000000002
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.026778597517136134
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "baselabs mammoth-1 (27B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h. The launcher and engine run in a docker container with network none (cut proven from inside before any item hash or model load: loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, read-only model and input, empty credential variables, HF offline flags, one request at a time, timed over loopback on the VM. Peak CUDA memory allocated after the scored run 59,375 MiB of 97,887 MiB. Measured 22 Oct 2026 (row 09:57-10:06Z, scored run 401 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720..., checked on the node and inside the container). This row's input was shredded from the node and its absence verified, and the VM was removed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 78.0375549583495,
    "I_sealed": 74.37139462333373,
    "base": 76.20447479084162,
    "excess": -6.173366435339936,
    "gap": 3.6661603350157748,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 88.89680314176312,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 88.67568417741427,
       "judge": 88.13447593935399,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 72.17046238785369,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 73.04539934543172,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 77.82599696969696,
       "hard": 80.89890462962963,
       "judge": 74.66825932203389,
       "standard": 52.51379999999999
      }
     },
     "sealed|choice": {
      "cc": 82.91172191752877,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 77.62229675031803,
       "judge": 82.37022526934379,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 72.33456301804218,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 75.86206896551724,
       "hard": 67.27272727272727,
       "judge": 75.38461538461539,
       "standard": 76.11940298507463
      }
     },
     "sealed|score": {
      "cc": 67.86789893443024,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 85.15065087719297,
       "hard": 64.31989746478874,
       "judge": 68.15543802083333,
       "standard": 65.89121727272727
      }
     }
    }
   },
   "jevbench_score": 75.37036297447123,
   "key": "mammoth-1",
   "last_measured_on": "2026-10-22",
   "licence": "MIT (model card metadata since 21 Oct 2026; no LICENSE file; base Qwen/Qwen3.5-27B is Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "baselabs/mammoth-1@03007ab3c4481bf01c71ef63adbbbc4b2093d71a",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.972972972972973,
    "decisive": 333,
    "decisive_rate": 0.8927613941018767,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 16,
   "ranked": true,
   "ranks": {
    "A": 16,
    "B": 10,
    "C": 15,
    "capability": 7
   },
   "repo": "https://huggingface.co/baselabs/mammoth-1",
   "scores": {
    "A": 75.37036297447123,
    "B": 75.5357213419485,
    "C": 75.37036297447123
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5256244310000057,
    "p50_s_raw": 0.18781221550000282,
    "p95_s_adjusted": 0.6893323799000428,
    "p95_s_raw": 0.2696661899500214
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The authors' own serving runtime: OneJev qev package at 81ce62f1597c91e46767d4d02d6ac2e18534fe94 plus the authors' serve.patch (sha256 f234c125f1d38d82104b9e1f24e75bde51e4401928acfba2fbf5c59ef2fb46b8; 15 qev/ files, manifest sha256 818ca0fc1d26ec6a2369aac210fb1de41d908a9e52b45e25bc6150714f046501), vendored as byte-pinned files (re-verified inside the smoke container: 15 files OK), no git clone, no pip -e, setup.sh never run. It is run through OUR first-party launcher baselabs_launcher.py (sha256 7a0896373a6677034909ffa56f17558cbd9ceaf389a9dafdea59cfc638ce26ed), which builds the authors' qev.engine.DecisionEngine (bfloat16 + fp32 LM head, block-bidirectional attention on the full-attention layers, slot_lm readout, 256 slots, debias 1, no calibration file so T=1, auto fork mode which ran as 'batched', author limits 32,768 / 65,536 tokens) and hands it to OneJev's own create_app on 127.0.0.1:8000. `qev serve` was NOT used: it routes these checkpoints (config has a vision_config) to MMDecisionEngine, which never attaches the block-bidirectional mask; the DecisionEngine is what the authors' own Decision Index runs use. The launcher adds only the authors' own 32BitIndexMath sequential-fork fallback and one read-only /_launcher/info route; no prompt, truncation, calibration, answer or usage change. After the scored run the engine reported noncausal attention with the mask installed and every counted mask build (3,000 of 3,000) carried block ids; the counters show that the mask was built with block ids, not numerical parity with the Decision Index runs or with a causal-mask comparison (neither was run). Code gate: cleared by the benchmark owner (22 Oct 2026 08:42Z); a later addendum covering the attention-mask monkeypatch, the Hub-kernel path and the setup.sh extras was disclosed to the owner afterwards and no fresh confirmation was claimed. DISCLOSED CODE PATTERNS (download/client/kernel paths are inactive; the attention-mask monkeypatch executes offline as described below): setup.sh (git clone, pip install -e, a CUDA extension build that can download a wheel), qev/client.py (urllib HTTP client class, unused), qev/engine.py and mm_engine.py Hub loaders (a local verified snapshot is passed, offline flags set), readout.py torch.load of a head file (checkpoints say slot_lm, no head file), constraints.txt listing wandb and an OpenTelemetry exporter stack (not installed), an in-process monkeypatch of transformers' create_causal_mask to add block ids (readout.py:104-124, runs inside the network-none container), and transformers' optional Hub-kernel path (inactive: the kernels package is not installed). A static source review and two independent runner reviews preceded the run (their one must was closed and re-checked before launch). Image: our own build on the pod (network on, before any sealed byte) from pytorch/pytorch:2.8.0-cuda12.8-cudnn9-devel@sha256:a7103283ea7113e10ae5d014bd2342acebda0bc53164b2f7b1dd6eb7a766bdb6 plus 50 hash-pinned wheels (--require-hashes, lock sha256 bf9fa61b272ce83fe5a2ac4c6db219484f929083d46efe0f0a5252ab969c31da, installed with --no-index --no-deps); torch 2.8.0+cu128, transformers 5.19.0, tokenizers 0.23.3, flash-linear-attention 0.5.2; causal-conv1d is not installed (transformers uses its torch fallback); row image sha256:598f1f31ed5aa27f455caefcc3a509e2140bcaf49cae685db419f8fecd5f4682. Weights: baselabs/mammoth-1 at 03007ab3c4481bf01c71ef63adbbbc4b2093d71a, the 10 serving files (54,733,723,313 bytes, serve manifest sha256 33025dc9...), fetched by us from the commit-pinned Hub URLs and sha256-verified file by file on the host and again in the smoke container; README, engine.py, setup.sh and the other author files were not used. FIRST MEASUREMENT (found by our own intake: Decision Index 0.4.1 entry; the GitHub request for this model came from a third party, not the authors). AUTHOR-DEFAULT CONTEXT LIMITS: the engine allows 32,768 tokens for state plus the longest question (strict >) and 65,536 per request and raises an error above that, which the authors' runtime returns through OneJev create_app as HTTP 422, never a truncation; the authors train at 16,384 tokens. The 15 items of 76,374-80,534 tokens were refused exactly as pre-registered and are scored wrong; the largest answered item is 1,854 tokens, inside the trained length. NOT AN AUTHOR-SHIPPED SERVER: the authors ship no server file; our first-party launcher around their DecisionEngine and OneJev's web app is what was timed (see code_revision). SERIAL ONLY: the server handles one request at a time and our driver sent one at a time; determinism over the first 50 positions was bit-identical (50/50, 0 argmax flips). causal-conv1d is not installed, so transformers' slower torch convolution fallback ran. LICENCE: MIT in the card metadata since 21 Oct 2026 (commit 9d1962f1 changed 'proprietary' to 'mit' and removed the evaluation-only line); there is no LICENSE file, no copyright holder and no carried Qwen attribution, and the Decision Index row still said 'proprietary' on 22 Oct (stale). Weights are identical to the Decision Index-scored revision 9d1962f1 (the earlier 68.87 row was another weight set and is not used). TRAINING DATA: not disclosed (the Decision Index notes it); the Decision Index states that its checks found signs that the model's training data overlapped one private new-domain test, which it does not count for this model, we cannot rule out overlap on other tests (our inference); the authors' technical report says decontamination against the Jev / Artificial Analysis index benchmarks was done (self-reported, not verified here); we ran no overlap check and no JevBench use or denial is declared. Our own public-versus-sealed check on this row: Intelligence 78.04 on the public items against 74.37 on the sealed ones, a gap of 3.67 points, so no penalty applies. SELF-REPORTED, unverified: Decision Index 0.4 score 71.24 and an Artificial Analysis replica score of 65.9 on the card; the DI lab's own Decision Index 0.4.1 full score is 68.44 (public 75.34, decision agreement with the submitted rows 99.85% on 831 rows against a 98.5% bar), DI-measured latency median 220 ms. 10 s RULE: max measured decision latency 5.78 s (one outlier; all other decisions <=0.49 s; median 0.19 s, p95 0.27 s); 0 decisions over 10 s; a 20k-token request took about 5.7 s warm and a 32k-token request about 17 s in the smoke, but no answered item is that large.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 95.74752222222223,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 95.5759
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 95.74752222222223
      },
      "calibration_gen": {
       "accuracy": 0.801980198019802,
       "cc": 68.72396086183281,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 68,
        "score": 63.622571875
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 68.72396086183281
      },
      "composite_gen": {
       "accuracy": 0.9108910891089109,
       "cc": 89.70989044194289,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 100,
        "score": 72.38170384615384
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 89.70989044194289
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.14344259259259,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 98.43032777777778
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.14344259259259
      },
      "jaggedness": {
       "accuracy": 0.75,
       "cc": 58.27060204744177,
       "cc_by_type": {
        "choice": 71.12608277189607,
        "noul": 35.13513513513513,
        "score": 68.55058823529411
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 58.27060204744177
      },
      "llm_router": {
       "accuracy": 0.9361022364217252,
       "cc": 84.22002665949906,
       "cc_by_type": {
        "choice": 93.66029761380645,
        "noul": 93.81443298969072,
        "score": 65.185349375
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 84.22002665949906
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.8148148148148148,
       "cc": 67.83860883652432,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 75,
        "score": 65.62922857142858
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 67.83860883652432
      },
      "multilingual": {
       "accuracy": 0.8254545454545454,
       "cc": 70.50534358550196,
       "cc_by_type": {
        "choice": 85.32768177372026,
        "noul": 55.55555555555556,
        "score": 70.63279342723004
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 70.50534358550196
      },
      "negation_pair_gen": {
       "accuracy": 0.9418604651162791,
       "cc": 85.3283288888889,
       "cc_by_type": {
        "choice": 100,
        "noul": 86.66666666666667,
        "score": 69.31832000000001
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 85.3283288888889
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.953125,
       "cc": 89.05839415184414,
       "cc_by_type": {
        "choice": 93.993993993994,
        "noul": 100,
        "score": 73.18118846153845
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 89.05839415184414
      },
      "usecase": {
       "accuracy": 0.8136363636363636,
       "cc": 73.00379157908498,
       "cc_by_type": {
        "choice": 77.94117647058823,
        "noul": 66.66666666666667,
        "score": 74.4035316
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 73.00379157908498
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 75.93615006105006,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": 46.856069230769236
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 75.93615006105006
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 84.03880666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 52.116419999999984
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 84.03880666666666
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 89.71216172887007,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 100,
        "score": 77.10108695652174
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 89.71216172887007
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 60.97645124146493,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 33.33333333333333,
        "score": 99.87535000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 60.97645124146493
      },
      "en": {
       "accuracy": 0.8618592528236316,
       "cc": 77.0320042250382,
       "cc_by_type": {
        "choice": 84.8427527123174,
        "noul": 75.53956834532374,
        "score": 70.71369161747343
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 77.0320042250382
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 76.33060235741931,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 38.77973529411766
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 76.33060235741931
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.55877777777776,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.34299999999998
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.55877777777776
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 85.39683890608876,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 98.54345789473685
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 85.39683890608876
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 78.93753282122906,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 75,
        "score": 78.57237500000002
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 78.93753282122906
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 64.9473619047619,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -5.15791428571426
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 64.9473619047619
      },
      "ja": {
       "accuracy": 0.8,
       "cc": 61.87464,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 25.623919999999988
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 61.87464
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 16.17297142857142,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 48.51891428571427
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 16.17297142857142
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.76368333333333,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.29105
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.76368333333333
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 30.77286023088024,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 51.62594000000003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 30.77286023088024
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 64.73543062130324,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 39.99999999999999,
        "score": 69.44116279069767
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 64.73543062130324
      },
      "pt": {
       "accuracy": 0.8571428571428571,
       "cc": 83.32287404591277,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100,
        "score": 71.18252903225806
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 83.32287404591277
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 43.79053510638298,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 60.00000000000001,
        "score": 47.96735000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 43.79053510638298
      },
      "tr": {
       "accuracy": 1,
       "cc": 94.68632916666667,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 84.0589875
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 94.68632916666667
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.58657849462365,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 95.0178
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.58657849462365
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 53.75864444444445,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 27.942600000000017
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 53.75864444444445
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8813333333333333,
       "cc": 84.67249660345587,
       "cc_by_type": {
        "choice": 84.67249660345587
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.67249660345587
      },
      "noul": {
       "accuracy": 0.864,
       "cc": 72.8,
       "cc_by_type": {
        "noul": 72.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 72.8
      },
      "score": {
       "accuracy": 0.792,
       "cc": 69.72138014440435,
       "cc_by_type": {
        "score": 69.72138014440435
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 69.72138014440435
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 49.03737058823529,
       "cc_by_type": {
        "choice": 84,
        "noul": 0,
        "score": 63.11211176470587
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 49.03737058823529
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 80.85620833731487,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 61.202165384615405
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 80.85620833731487
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 32.81459159891599,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": -69.0359
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 32.81459159891599
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 12.519774000974659,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": -33.333333333333336,
        "score": 29.372187500000013
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 12.519774000974659
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 59.614744112318846,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 74.49640625
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.614744112318846
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 30.680094654088066,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 66.66666666666667,
        "score": -49.46914999999996
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 30.680094654088066
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 65.21579784529196,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 82.2020153846154
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 65.21579784529196
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 60.91103279806091,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 50,
        "score": 53.639021739130435
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 60.91103279806091
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 57.01100740740741,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 19.999999999999996,
        "score": 71.86635555555556
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 57.01100740740741
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 87.62383869314125,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100,
        "score": 75.71948181818182
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 87.62383869314125
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 80.56180541852633,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 75.53554545454546
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 80.56180541852633
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 36.799901433691765,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 41.582500000000024
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 36.799901433691765
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 62.91369907407407,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 68.74109722222221
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 62.91369907407407
      },
      "model_routing": {
       "accuracy": 0.9170506912442397,
       "cc": 79.61527939664502,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 90.90909090909092,
        "score": 53.78090312499999
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.61527939664502
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 24.215866617741153,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 95.6802294117647
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 24.215866617741153
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 60.37083188105612,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 81.20534615384616
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 60.37083188105612
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 64.23455776103545,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50,
        "score": 72.26032352941176
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 64.23455776103545
      },
      "scientific_discovery": {
       "accuracy": 0.782608695652174,
       "cc": 72.78247748792269,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100,
        "score": 61.82569333333332
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 72.78247748792269
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.86276578947367,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.58829736842104
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.86276578947367
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      89.92645950017267,
      94.05049640628478
     ],
     "I": [
      72.3587407266138,
      79.89757564645463
     ],
     "capability": [
      81.55475983073057,
      86.27316534238307
     ],
     "composite": null,
     "seed": 51
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 94.26152926942419,
     "capability": 85.2330020301329,
     "intelligence": 76.20447479084162
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8813333333333333,
      "calibration": 95.01703213114757,
      "cc": 84.67249660345587,
      "cc_by_type": {
       "choice": 84.67249660345587
      },
      "cc_tiered": 85.90426252964593,
      "cc_tiered_open": 88.89680314176312,
      "cc_tiered_sealed": 82.91172191752877,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.67249660345587
     },
     "noul": {
      "accuracy": 0.864,
      "calibration": 94.5443587131367,
      "cc": 72.8,
      "cc_by_type": {
       "noul": 72.8
      },
      "cc_tiered": 72.25251270294794,
      "cc_tiered_open": 72.17046238785369,
      "cc_tiered_sealed": 72.33456301804218,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 72.8
     },
     "score": {
      "accuracy": 0.792,
      "calibration": 93.22319696398831,
      "cc": 69.72138014440435,
      "cc_by_type": {
       "score": 69.72138014440435
      },
      "cc_tiered": 70.45664913993099,
      "cc_tiered_open": 73.04539934543172,
      "cc_tiered_sealed": 67.86789893443024,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 69.72138014440435
     }
    },
    "raw_on_A": {
     "calibration": 91.72815618560855,
     "capability": 83.14098848301494,
     "intelligence": 74.55382078042132
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.65061323888425,
    "Emphasis on Accuracy 60:20:20": 72.77209571376169,
    "Emphasis on Cost 20:20:60": 64.55937657064973,
    "Emphasis on Speed 20:60:20": 75.57819877417623,
    "Intelligence only": 76.20447479084162
   }
  },
  {
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "oraculumai",
   "axes": {
    "calibration": 84.0077997110931,
    "cost": 78.87189352613305,
    "intelligence": 32.54303915979197,
    "speed": 90.57533514329342
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07560407620955467,
      "ece_score": 84.87918475808907,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 84.87918475808907,
      "score_before_time_limit": 84.87918475808907,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.37735849056603776,
      "brier": 0.1368158290573463,
      "ece": 0.05776783172439211,
      "n": 371,
      "score": 88.44643365512158,
      "score_before_time_limit": 88.44643365512158,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9051490514905149,
      "interval90_width": 0.4345076784101176,
      "n": 369,
      "nrps": 0.12172060479348325,
      "score": 78.69778072006868,
      "score_before_time_limit": 78.69778072006868,
      "time_limit_failed_share": 0,
      "top_ece": 0.1521618904025716
     }
    },
    "score": 84.0077997110931
   },
   "capability": 58.27541943544253,
   "class": "system-one-open",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 836,500 input + 0 output tokens over 1,624 decisions = 515.086206897 tokens/decision. GPU-time cost USD 0.00506139481438631/1,000; old list-price cost USD 0.0154525862068966/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00506139481438631/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00506139481438631
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Manchego v2.1",
   "endpoint_condition": "evaluator-owned offline RTX6000 GPU",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 40.1322357986733,
    "I_sealed": 26.800244420595554,
    "base": 33.46624010963443,
    "excess": 10.758603735639502,
    "gap": 13.331991378077745,
    "penalty": 0.972413962643605,
    "per_type_split": {
     "open|choice": {
      "cc": 60.640330926520605,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 45.26580685750237,
       "judge": 68.35860250494396,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 19.02691511387163,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 13.043478260869556,
       "judge": 0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 40.72946135562766,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.415593474232956,
       "hard": 33.07729129866341,
       "judge": 27.484035794573337,
       "standard": 64.558873751835
      }
     },
     "sealed|choice": {
      "cc": 56.6033475264085,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 49.60407239819003,
       "judge": 56.238361266294234,
       "standard": 66.7446729892844
      }
     },
     "sealed|noul": {
      "cc": 2.9507869907230795,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -30.097087378640776,
       "judge": 13.157894736842103,
       "standard": 35.2112676056338
      }
     },
     "sealed|score": {
      "cc": 20.84659874465509,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 52.898828557922364,
       "hard": 20.484461869906934,
       "judge": 15.149834013146535,
       "standard": 14.089904684780608
      }
     }
    }
   },
   "jevbench_score": 25.53791020495781,
   "key": "manchego-v21-v15",
   "last_measured_on": "2026-10-04",
   "licence": "Apache-2.0 (code and weights, author-declared)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9047619047619048,
    "decisive_rate": 0.6226415094339622,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.0077997110931,
     "cost": 64.32996474661243,
     "intelligence": 32.54303915979197,
     "speed": 90.7347510499745
    },
    "board_release": "v1.6.43",
    "capability": 58.27541943544253,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 64,
    "capability_rank": 89,
    "composite_ci95": {
     "A": [
      17.54626281581445,
      31.019351182791876
     ],
     "B": [
      14.871831849106172,
      27.358180563302277
     ],
     "C": [
      12.18490473320448,
      21.54121609916103
     ]
    },
    "jevbench_score": 24.487333472500133,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 81,
    "rank": 114,
    "ranks": {
     "A": 114,
     "B": 124,
     "C": 129,
     "capability": 142
    },
    "scores": {
     "A": 24.487333472500133,
     "B": 21.196520189880484,
     "C": 17.005092689236207
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2574284589994932,
     "p50_s_raw": 0.05371422949974658,
     "p95_s_adjusted": 0.32799584188469455,
     "p95_s_raw": 0.08899792094234726
    },
    "views": {
     "Balanced 33:33:33": 22.181162996647437,
     "Emphasis on Accuracy 60:20:20": 17.83638056849814,
     "Emphasis on Cost 20:20:60": 23.964666214390906,
     "Emphasis on Speed 20:60:20": 26.697557732929795,
     "Intelligence only": 13.785874409327832
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 178,
   "ranked": true,
   "ranks": {
    "A": 178,
    "B": 179,
    "C": 180,
    "capability": 175
   },
   "repo": "https://github.com/nschlaepfer/manchego-serve",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 25.53791020495781,
    "B": 21.818063924375018,
    "C": 17.73465986455404
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26150697499397213,
    "p50_s_raw": 0.055753487496986054,
    "p95_s_adjusted": 0.33495248848921616,
    "p95_s_raw": 0.09247624424460808
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author server manchego-serve v0.1.1 @ 0ee62188af474e3c1d11d47a9c4c8b3a08766db3; oraculumai/Manchego @ 77403228b7dfdf823af99a5f562bdcf80b708d4c. Evaluator-owned RTX6000, offline/read-only container, safetensors with verified weight and support-file checksums, bfloat16, sequential execution, temperature 1.0 and original option order. Stock TypeSafe adapter; all three types native. The author disclosed using public aggregate/family results during development, a prior Jev-assisted phrase filter on two template pools, and one Jev-labelled development group for checkpoint selection. These disclosures do not establish item-text leakage; this is not a claim of fully independent development. Cost is an estimate using measured token usage and the frozen Qwen3.5-4B reference, not a bookable Manchego tariff.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.8981045039492,
      85.26223280687248
     ],
     "I": [
      28.50329098429543,
      35.95139455461947
     ],
     "capability": [
      54.23896847458738,
      59.77267571842297
     ],
     "composite": null,
     "seed": 65
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.0077997110931,
     "capability": 58.27541943544253,
     "intelligence": 32.54303915979197
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6693333333333333,
      "calibration": 84.87918475808907,
      "cc": 57.712456068887754,
      "cc_by_type": {
       "choice": 57.712456068887754
      },
      "cc_tiered": 58.62183922646455,
      "cc_tiered_open": 60.640330926520605,
      "cc_tiered_sealed": 56.6033475264085,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.712456068887754
     },
     "noul": {
      "accuracy": 0.5573333333333333,
      "calibration": 88.44643365512158,
      "cc": 11.466666666666669,
      "cc_by_type": {
       "noul": 11.466666666666669
      },
      "cc_tiered": 10.988851052297356,
      "cc_tiered_open": 19.02691511387163,
      "cc_tiered_sealed": 2.9507869907230795,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 11.466666666666669
     },
     "score": {
      "accuracy": 0.49066666666666664,
      "calibration": 78.69778072006868,
      "cc": 26.44898115857113,
      "cc_by_type": {
       "score": 26.44898115857113
      },
      "cc_tiered": 30.788030050141376,
      "cc_tiered_open": 40.72946135562766,
      "cc_tiered_sealed": 20.84659874465509,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 26.44898115857113
     }
    },
    "raw_on_A": {
     "calibration": 84.55956989627305,
     "capability": 59.747806252757606,
     "intelligence": 34.936042609242165
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 23.340849590714303,
    "Emphasis on Accuracy 60:20:20": 18.27444897680165,
    "Emphasis on Cost 20:20:60": 26.540784078317966,
    "Emphasis on Speed 20:60:20": 27.677101086504035,
    "Intelligence only": 13.785874409327832
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Inception (API, free on OpenRouter) - measurement request, 6 Oct 2026",
   "axes": {
    "calibration": 81.34341651168008,
    "cost": 62.08313746631354,
    "intelligence": 65.90701345473542,
    "speed": 84.81995518783543
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11045316563089375,
      "ece_score": 77.90936687382126,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 77.90936687382126,
      "score_before_time_limit": 77.90936687382126,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.10242587601078168,
      "brier": 0.10151182638394682,
      "ece": 0.08813961357701941,
      "n": 371,
      "score": 82.37207728459612,
      "score_before_time_limit": 82.37207728459612,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8753387533875339,
      "interval90_width": 0.1458897922312557,
      "n": 369,
      "nrps": 0.06239733540901056,
      "score": 83.74880537662287,
      "score_before_time_limit": 83.74880537662287,
      "time_limit_failed_share": 0,
      "top_ece": 0.13131327852926614
     }
    },
    "score": 81.34341651168008
   },
   "capability": 73.62521498320774,
   "class": "Hosted decision API",
   "composite_ci95": {
    "A": [
     69.82880671245154,
     73.74963246907883
    ],
    "B": [
     67.70690015261091,
     72.8950112029009
    ],
    "C": [
     69.63070886080774,
     73.74963246907883
    ]
   },
   "cost": {
    "basis": "LABELLED ESTIMATE, not charged - and nothing WAS charged: usage.cost is 0 on every one of the 1,500 calls. Inception publishes NO paid tariff for Mercury Decide: OpenRouter lists it free-only (prompt 0, completion 0) and the non-free slug inception/mercury-decide has no endpoints at all (404 'No endpoints found', read 7 Oct 2026), so there is nothing bookable. Because the system is API-only with undisclosed weights, the open rows' size-class estimate table cannot apply either. The row is therefore priced from THE SAME OPERATOR'S cheapest published per-token tariff on the SAME platform, inception/mercury-2.5 at USD 0.04 per 1M input and USD 0.15 per 1M output, which IS a key in the frozen 25 Sep snapshot - x this row's own measured tokens. Output tokens are 3 per decision on average (4,434 over 1,477 answers) because the probability is read from the model rather than written out, so the output rate is nearly inert and the Cost axis is set by the input rate alone. PRICING REFERENCE NOTE: the rate inception/mercury-2.5 (0.04, 0.15) is already IN the dated snapshot; what is missing is only a base-model decision line pointing at it, so pricing_v15.reference() raises on every inception/* string. receipts/COST-ALTERNATIVES-R58.json has the two alternatives' arithmetic already done.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.01807942666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.018360961408259987
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Mercury Decide (Inception; System One decisions API, served free on OpenRouter as inception/mercury-decide:free)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), COMPLETE: 1,500 of 1,500 dispatched, 1,477 answered, 23 invalid. Measured through the reviewed run_api_full_v161.py UNCHANGED, on the published 5 Oct 2026 amendment (hosted systems answer the same full S u P as self-hosted ones); provider inception, proxy openrouter, every sealed item exposure-logged before the first request. measurement request of 6 Oct 2026 ('put it on JevBench', under https://x.com/airesearch12/status/2107569862486659111). NO MAPPING DECISION OF OURS: Mercury Decide publishes the same /v1/systemone contract as Jev and all three primitives come back in the canonical shape (noul a float; choice a label plus probabilities keyed by the item's own option labels; score probabilities keyed by level index '0'..'n-1' - 0-based, unlike ClassOne's), so the vendored typesafe adapter's request body, answer mapping and validations are reused UNCHANGED. or_decisions_r58.OpenRouterDecisionsAdapter rewrites ONLY the URL, because OpenRouter serves the contract at POST /api/alpha/decisions rather than /v1/systemone - its own 400 from chat/completions names that path - and it does so by calling the parent run(), so none of the reviewed mapping is duplicated. The pinned harness/vendor tree was NOT edited; the module is reached by PYTHONPATH. THE 23 LONG ITEMS: on this pool exactly 23 of 1,500 items are 77,857-82,318 tokens and the 24th largest is below 2,048 (run 57's own per-item counts on the identical pool, receipt LONGITEM-TOKENS-R58.json). They are also the 23 items the v1.6.1 cost rule excludes. THIS ROW REFUSES THEM: the context window is 32,768 tokens and all 23 came back as HTTP 422 with the provider's own 'upstream_error: Decision service could not complete the request' - a disclosed refusal, which the harness counts as the system's own rather than as an outage. The invalid set is EXACTLY those 23 and nothing else failed in 1,477 answers. RUN IN TWO PASSES BECAUSE OF A FREE-TIER BURST LIMIT, AND IT DOES NOT MOVE THE SPEED FIGURE: items 1-736 ran back to back at ~1.7 req/s until OpenRouter returned a header-less 429 ('rate_limit_exceeded: Request or spending limit exceeded'), which is run_v16's own stop rule; the row was resumed at 1 req/s and completed. The two passes' medians differ by 0.038 s (0.3624 s vs 0.3246 s over 1,477 answers; 0.3624 vs 0.3104 on the 124 items the Speed axis actually uses), and run_v16 sleeps the spacing BEFORE starting the timer, so the pacing is not in the measurement. Receipt LATENCY-SEGMENTS-mercury-R58.txt. Production API, so NO latency adjustment. Serial, one decision per request, no retry, no batching. add-requests run 58",
   "endpoint_kind": "api",
   "full_coverage": true,
   "gpu": null,
   "intelligence": {
    "I_open": 67.12906498043914,
    "I_sealed": 64.68496192903172,
    "base": 65.90701345473542,
    "excess": -0.12928459103081957,
    "gap": 2.4441030514074242,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 79.71168908755723,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 81.12614029569046,
       "judge": 80.22412656558997,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 58.71014492753623,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 30.000000000000004,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 62.96536092622397,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.561695179434636,
       "hard": 63.165822850913635,
       "judge": 76.80814242854491,
       "standard": 42.0020976967579
      }
     },
     "sealed|choice": {
      "cc": 74.09139259954716,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 77.38526346907102,
       "hard": 68.43891402714931,
       "judge": 79.51582867783985,
       "standard": 75.61276019214189
      }
     },
     "sealed|noul": {
      "cc": 57.27197132719671,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 35.92233009708738,
       "judge": 65.78947368421053,
       "standard": 71.83098591549295
      }
     },
     "sealed|score": {
      "cc": 62.691521860351266,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 69.70067055435705,
       "hard": 51.597448628789465,
       "judge": 70.50916076191972,
       "standard": 69.64863562411932
      }
     }
    }
   },
   "jevbench_score": 72.24968682544169,
   "key": "mercury-decide",
   "last_measured_on": "2026-10-07",
   "licence": null,
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "provider-reported measurement model slug: inception/mercury-decide:free",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9039039039039038,
    "decisive_rate": 0.8975741239892183,
    "supported": true,
    "valid": 371
   },
   "open": "closed weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 11,
    "axes": {
     "calibration": 81.34341651168008,
     "cost": 62.08313746631354,
     "intelligence": 65.90701345473542,
     "speed": 85.47752366478576
    },
    "board_release": "v1.6.43",
    "capability": 73.62521498320774,
    "capability_api_board_rank": 13,
    "capability_open_board_rank": null,
    "capability_rank": 27,
    "composite_ci95": {
     "A": [
      69.93954261051735,
      73.87316393178861
     ],
     "B": [
      67.79015712351674,
      72.99152541270097
     ],
     "C": [
      69.74081685972264,
      73.87316393178861
     ]
    },
    "jevbench_score": 72.36824049637728,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 15,
    "ranks": {
     "A": 15,
     "B": 15,
     "C": 15,
     "capability": 51
    },
    "scores": {
     "A": 72.36824049637728,
     "B": 70.97659528968215,
     "C": 72.36824049637728
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.33213604614138603,
     "p50_s_raw": 0.33213604614138603,
     "p95_s_adjusted": 0.8529658075422044,
     "p95_s_raw": 0.8529658075422044
    },
    "views": {
     "Balanced 33:33:33": 69.80103192778668,
     "Emphasis on Accuracy 60:20:20": 68.18948202584015,
     "Emphasis on Cost 20:20:60": 66.49451324072818,
     "Emphasis on Speed 20:60:20": 75.32698952826067,
     "Intelligence only": 65.90701345473542
    }
   },
   "provenance": {
    "carried_metadata_from": "Current-pool API measurement registry",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 52,
   "ranked": true,
   "ranks": {
    "A": 52,
    "B": 45,
    "C": 41,
    "capability": 69
   },
   "repo": "https://openrouter.ai/inception/mercury-decide:free",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 72.24968682544169,
    "B": 70.88533274889458,
    "C": 72.24968682544169
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.34386441484093666,
    "p50_s_raw": 0.34386441484093666,
    "p95_s_adjusted": 0.9585554625838959,
    "p95_s_raw": 0.9585554625838959
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.84031884046848,
      84.00969333082946
     ],
     "I": [
      60.19803361460591,
      70.00098427505237
     ],
     "capability": [
      69.08542722449891,
      76.48291495498401
     ],
     "composite": {
      "A": [
       69.82880671245154,
       73.74963246907883
      ],
      "B": [
       67.70690015261091,
       72.8950112029009
      ],
      "C": [
       69.63070886080774,
       73.74963246907883
      ]
     },
     "seed": 91
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.34341651168008,
     "capability": 73.62521498320774,
     "intelligence": 65.90701345473542
    },
    "per_type": {
     "choice": {
      "accuracy": 0.808,
      "calibration": 77.90936687382126,
      "cc": 75.44594223354774,
      "cc_by_type": {
       "choice": 75.44594223354774
      },
      "cc_tiered": 76.9015408435522,
      "cc_tiered_open": 79.71168908755723,
      "cc_tiered_sealed": 74.09139259954716,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.44594223354774
     },
     "noul": {
      "accuracy": 0.8026666666666666,
      "calibration": 82.37207728459612,
      "cc": 60.53333333333333,
      "cc_by_type": {
       "noul": 60.53333333333333
      },
      "cc_tiered": 57.99105812736647,
      "cc_tiered_open": 58.71014492753623,
      "cc_tiered_sealed": 57.27197132719671,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 60.53333333333333
     },
     "score": {
      "accuracy": 0.712,
      "calibration": 83.74880537662287,
      "cc": 62.21714605598172,
      "cc_by_type": {
       "score": 62.21714605598172
      },
      "cc_tiered": 62.82844139328762,
      "cc_tiered_open": 62.96536092622397,
      "cc_tiered_sealed": 62.691521860351266,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.21714605598172
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.65404525909453,
    "Emphasis on Accuracy 60:20:20": 68.1052419126018,
    "Emphasis on Cost 20:20:60": 66.41440648974917,
    "Emphasis on Speed 20:60:20": 75.01947411259273,
    "Intelligence only": 65.90701345473542
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Messier AI (@agentmessier-ai, agentmessier.ai@gmail.com)",
   "axes": {
    "calibration": 73.90972729526267,
    "cost": 79.42950661954919,
    "intelligence": 42.35139759222662,
    "speed": 92.56000105620268
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.16098163180524172,
      "ece_score": 67.80367363895165,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 67.80367363895165,
      "score_before_time_limit": 67.80367363895165,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.15200000000000002,
      "brier": 0.14568508389123155,
      "ece": 0.10030106317015194,
      "n": 375,
      "score": 79.93978736596961,
      "score_before_time_limit": 79.93978736596961,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8533333333333334,
      "interval90_width": 0.378888888888889,
      "n": 375,
      "nrps": 0.12363172157496395,
      "score": 73.98572088086677,
      "score_before_time_limit": 73.98572088086677,
      "time_limit_failed_share": 0,
      "top_ece": 0.1983269304038504
     }
    },
    "score": 73.90972729526267
   },
   "capability": 58.130562443744644,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 727,432 input + 1,477 output tokens over 1,477 decisions = 493.506431957 tokens/decision. GPU-time cost USD 0.00484934533700964/1,000; old list-price cost USD 0.0149251929587001/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00484934533700964/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.0184433
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimated",
    "usage_estimated_rows": 1500,
    "usd_per_1000": 0.004849345337009636
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Messier One v0.2 (Qwen3.5-4B fine-tune, one prefill, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. REPLACES the v0.1 request (GitHub #194, measured by run 54); the submission names it so. Weights agentmessier/messier-one tag v0.2 = 8bbddb84140e8fc732b35713f998ed628fc70306 (CC BY-NC 4.0, NON-COMMERCIAL), front end github.com/agentmessier-ai/messier-one tag v0.2 = 1b8c3b80 (Apache-2.0). README install line (vllm==0.30.0, transformers==5.18.0; torch 2.13.0+cu130 resolved by vllm) and the author's two serve commands verbatim; --gate off, their default and the configuration of their own reported single-read run. The author discloses that part of the training material was written in the families of JevBench's hard tier, with no JevBench item used. Their Prompter caps the state at 8,000 tokens (max_total 8,600) and splices long states around a readable marker, so the 23 long items are answered truncated (r54 s on disclosed truncation). UNCHANGED typesafe adapter; the smoke gate refused the first attempt at usage-input-tokens and passed with its existing pre-registered opt-in. One RTX 5090 32 GB (Lium, France). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 455A3944",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 42.671434486238354,
    "I_sealed": 42.03136069821489,
    "base": 42.35139759222662,
    "excess": -1.9333138544147794,
    "gap": 0.6400737880234644,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 54.85241164824772,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 57.189189189189186,
       "hard": 60.36489462094999,
       "judge": 36.71720500988794,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 42.58109040717736,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 0,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 30.580801403290003,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.135231705449634,
       "hard": 32.351343308274984,
       "judge": 27.483794440784738,
       "standard": 22.40801288599812
      }
     },
     "sealed|choice": {
      "cc": 55.440313317505925,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 73.3944276106718,
       "hard": 51.64027149321266,
       "judge": 54.37616387337058,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 41.10238724117829,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72,
       "hard": 22.330097087378633,
       "judge": 44.736842105263165,
       "standard": 57.746478873239425
      }
     },
     "sealed|score": {
      "cc": 29.55138153596046,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.40617006718976,
       "hard": 32.98321832976057,
       "judge": 19.789254292160475,
       "standard": 18.90350454844556
      }
     }
    }
   },
   "jevbench_score": 47.407255888808336,
   "key": "messier-one-v0.2",
   "last_measured_on": "2026-10-07",
   "licence": "cc-by-nc-4.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "agentmessier/messier-one@8bbddb84140e8fc732b35713f998ed628fc70306",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8522012578616353,
    "decisive_rate": 0.848,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.90972729526267,
     "cost": 64.78240136302922,
     "intelligence": 42.35139759222662,
     "speed": 92.70909408602279
    },
    "board_release": "v1.6.43",
    "capability": 58.130562443744644,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 65,
    "capability_rank": 90,
    "composite_ci95": {
     "A": [
      29.542188267758267,
      57.66843444819182
     ],
     "B": [
      26.199389362563995,
      53.35068096010408
     ],
     "C": [
      20.51540851927657,
      40.047523922355424
     ]
    },
    "jevbench_score": 45.29059491870636,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 45,
    "rank": 70,
    "ranks": {
     "A": 70,
     "B": 74,
     "C": 74,
     "capability": 143
    },
    "scores": {
     "A": 45.29059491870636,
     "B": 41.2441782613488,
     "C": 31.451802026879417
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20452698896988294,
     "p50_s_raw": 0.027263494484941475,
     "p95_s_adjusted": 0.26202333298570013,
     "p95_s_raw": 0.056011666492850054
    },
    "views": {
     "Balanced 33:33:33": 43.19017554763209,
     "Emphasis on Accuracy 60:20:20": 36.959944674970394,
     "Emphasis on Cost 20:20:60": 44.448060509071624,
     "Emphasis on Speed 20:60:20": 50.236730867619606,
     "Intelligence only": 30.38527918499109
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 124,
   "ranked": true,
   "ranks": {
    "A": 124,
    "B": 129,
    "C": 129,
    "capability": 176
   },
   "repo": "https://huggingface.co/agentmessier/messier-one",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 47.407255888808336,
    "B": 42.63085471503479,
    "C": 32.92170547833912
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20892030798713676,
    "p50_s_raw": 0.029460153993568383,
    "p95_s_adjusted": 0.26547231497825124,
    "p95_s_raw": 0.05773615748912562
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.69942429480604,
      76.95757379447907
     ],
     "I": [
      35.53670432007418,
      46.90962732194912
     ],
     "capability": [
      53.17607857615277,
      61.1523915096487
     ],
     "composite": null,
     "seed": 93
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.90972729526267,
     "capability": 58.130562443744644,
     "intelligence": 42.35139759222662
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6586666666666666,
      "calibration": 67.80367363895165,
      "cc": 56.348341748529286,
      "cc_by_type": {
       "choice": 56.348341748529286
      },
      "cc_tiered": 55.146362482876825,
      "cc_tiered_open": 54.85241164824772,
      "cc_tiered_sealed": 55.440313317505925,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 56.348341748529286
     },
     "noul": {
      "accuracy": 0.7226666666666667,
      "calibration": 79.93978736596961,
      "cc": 44.53333333333334,
      "cc_by_type": {
       "noul": 44.53333333333334
      },
      "cc_tiered": 41.841738824177824,
      "cc_tiered_open": 42.58109040717736,
      "cc_tiered_sealed": 41.10238724117829,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 44.53333333333334
     },
     "score": {
      "accuracy": 0.504,
      "calibration": 73.98572088086677,
      "cc": 32.623848126624935,
      "cc_by_type": {
       "score": 32.623848126624935
      },
      "cc_tiered": 30.06609146962523,
      "cc_tiered_open": 30.580801403290003,
      "cc_tiered_sealed": 29.55138153596046,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 32.623848126624935
     }
    },
    "raw_on_A": {
     "calibration": 79.87142853291357,
     "capability": 63.87968579812095,
     "intelligence": 47.887943063328336
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 45.789674971611724,
    "Emphasis on Accuracy 60:20:20": 38.06962624032526,
    "Emphasis on Cost 20:20:60": 49.69558901103812,
    "Emphasis on Speed 20:60:20": 52.28270727133392,
    "Intelligence only": 30.38527918499109
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Messier AI (@agentmessier-ai)",
   "axes": {
    "calibration": 72.86180997007926,
    "cost": 75.66359272969224,
    "intelligence": 52.02542301918905,
    "speed": 89.42792459288492
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12514095842876055,
      "ece_score": 74.9718083142479,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 74.9718083142479,
      "score_before_time_limit": 74.9718083142479,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.17675601582379594,
      "ece": 0.15402190415708572,
      "n": 375,
      "score": 69.19561916858285,
      "score_before_time_limit": 69.19561916858285,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.856,
      "interval90_width": 0.33844444444444444,
      "n": 375,
      "nrps": 0.11129842749409077,
      "score": 74.41800242740706,
      "score_before_time_limit": 74.41800242740706,
      "time_limit_failed_share": 0,
      "top_ece": 0.2001707619788841
     }
    },
    "score": 72.86180997007926
   },
   "capability": 62.443616494634156,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 978,475 input + 0 output tokens over 1,485 decisions = 658.905723906 tokens/decision. GPU-time cost USD 0.00647460943332089/1,000; old list-price cost USD 0.0197671717171717/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00647460943332089/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.022117700000000004
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.006474609433320888
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Messier One v0.4 (Qwen3.5-4B fine-tune, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-14",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX A6000 48 GB - different hardware from the H100/RTX 5090 rows of this draw and from the RTX 3090 the author quotes. Started by the author's OWN ./serve_vllm.sh MODEL_DIR 8080 with its defaults, which is what the author insists on for v0.4: vLLM 0.30.0, --max-model-len 8192, chunked prefill and prefix caching OFF (this revision is non-causal), --max-logprobs 256, processed_logprobs, front end --max-state 7500 --max-total 8150 --gate 0.5 --noul-floor 0.801; transformers 5.18.0 exactly as pinned. One request at a time from the unchanged runner. Isolation: the server ran inside a network namespace with loopback only (unshare -n), the cut proven before the sealed input was staged and re-proven immediately before the first sealed request, with the synthetic smoke gate passed in between; a network namespace replaces docker --network none because the pod carried no local image (under the pre-registration). The pod host kept its network for ssh.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 55.631843562186205,
    "I_sealed": 48.419002476191885,
    "base": 52.02542301918905,
    "excess": -2.626685684361391,
    "gap": 7.21284108599432,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 72.70061629878168,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 75.46398238439761,
       "judge": 64.40342781806197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 59.21394064872326,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 30.000000000000004,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 34.98097373905368,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 43.1256839171765,
       "hard": 43.01475509215395,
       "judge": 34.883418864196,
       "standard": 14.987388256078226
      }
     },
     "sealed|choice": {
      "cc": 60.22807849887394,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 56.80582861107898,
       "judge": 54.162585700293846,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 51.03745559638509,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 68.96551724137932,
       "hard": 50.90909090909091,
       "judge": 38.46153846153846,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 33.991473333316634,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 53.18191088973121,
       "hard": 30.414088756811818,
       "judge": 29.47815246036215,
       "standard": 38.32100501755069
      }
     }
    }
   },
   "jevbench_score": 69.75377301204921,
   "key": "messier-one-v0.4",
   "last_measured_on": "2026-10-14",
   "licence": "Weights CC BY-NC 4.0 (non-commercial: free for research and evaluation, commercial licence on request; listing a measured score is evaluation use); code Apache-2.0; base Qwen/Qwen3.5-4B Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.22",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "agentmessier/messier-one@14c84de57b71bf48608e2d886d86ebc5c6f8897e",
   "noul_decisive": {
    "acc_among_decisive": 0.7786666666666666,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 72.86180997007926,
     "cost": 61.121663950171616,
     "intelligence": 52.02542301918905,
     "speed": 89.67127141858641
    },
    "board_release": "v1.6.43",
    "capability": 62.443616494634156,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 49,
    "capability_rank": 74,
    "composite_ci95": {
     "A": [
      54.231114077287856,
      68.26806273860754
     ],
     "B": [
      50.59939794983643,
      65.50718704517482
     ],
     "C": [
      37.66049588700546,
      60.89651165228654
     ]
    },
    "jevbench_score": 66.16093508471093,
    "measured_in": "v1.6.22",
    "method": "v1.6",
    "open_board_rank": 24,
    "rank": 45,
    "ranks": {
     "A": 45,
     "B": 48,
     "C": 51,
     "capability": 116
    },
    "scores": {
     "A": 66.16093508471093,
     "B": 62.75099656762108,
     "C": 49.74281676276681
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2828623611479998,
     "p50_s_raw": 0.06643118057399988,
     "p95_s_adjusted": 0.3813271221704781,
     "p95_s_raw": 0.11566356108523905
    },
    "views": {
     "Balanced 33:33:33": 64.19305626178894,
     "Emphasis on Accuracy 60:20:20": 58.70145093265181,
     "Emphasis on Cost 20:20:60": 62.92818786712826,
     "Emphasis on Speed 20:60:20": 72.42418258466023,
     "Intelligence only": 52.02542301918905
    }
   },
   "rank": 81,
   "ranked": true,
   "ranks": {
    "A": 81,
    "B": 92,
    "C": 97,
    "capability": 147
   },
   "repo": "https://huggingface.co/agentmessier/messier-one",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.75377301204921,
    "B": 65.30318824170864,
    "C": 52.44407662931919
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2880231073126197,
    "p50_s_raw": 0.06901155365630984,
    "p95_s_adjusted": 0.396077533904463,
    "p95_s_raw": 0.1230387669522315
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 82.16341854240056,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 71.49025562720168
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 82.16341854240056
      },
      "calibration_gen": {
       "accuracy": 0.5148514851485149,
       "cc": 29.04494301665566,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": 43.99999999999999,
        "score": 9.942888434908658
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 29.04494301665566
      },
      "composite_gen": {
       "accuracy": 0.6435643564356436,
       "cc": 46.36869268140659,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": 55.55555555555556,
        "score": 9.566782651265848
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 46.36869268140659
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.14562852488994,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 89.4368855746698
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.14562852488994
      },
      "jaggedness": {
       "accuracy": 0.5611111111111111,
       "cc": 33.94067092331888,
       "cc_by_type": {
        "choice": 49.4706448508181,
        "noul": 24.32432432432432,
        "score": 28.027043594814227
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 33.94067092331888
      },
      "llm_router": {
       "accuracy": 0.7795527156549521,
       "cc": 66.83382437622242,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 75.25773195876289,
        "score": 56.14979030259788
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 66.83382437622242
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": 5.043826168411282,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": 0,
        "score": 5.454059150395141
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 5.043826168411282
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": 10.394403866583607,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": 25,
        "score": -7.218850255919285
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 10.394403866583607
      },
      "multilingual": {
       "accuracy": 0.6909090909090909,
       "cc": 53.25250361364891,
       "cc_by_type": {
        "choice": 66.7427453537659,
        "noul": 55.55555555555556,
        "score": 37.459209931625296
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 53.25250361364891
      },
      "negation_pair_gen": {
       "accuracy": 0.6395348837209303,
       "cc": 24.730034373509014,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 26.66666666666666,
        "score": -19.143230212806284
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 24.730034373509014
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 80.45660602129784,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100,
        "score": 62.39083908491454
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.45660602129784
      },
      "usecase": {
       "accuracy": 0.65,
       "cc": 52.043880329569305,
       "cc_by_type": {
        "choice": 62.00980392156864,
        "noul": 58.33333333333333,
        "score": 35.788503733805946
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 52.043880329569305
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 17.790541563258085,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 100,
        "score": -51.39028007213051
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 17.790541563258085
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 59.56093069774181,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 45.34945875989212
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 59.56093069774181
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 72.21420782877351,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 75,
        "score": 57.57182702614354
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 72.21420782877351
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 54.75552840589854,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 97.78613829032132
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 54.75552840589854
      },
      "en": {
       "accuracy": 0.6828844483058211,
       "cc": 51.49058721733788,
       "cc_by_type": {
        "choice": 63.754408659889414,
        "noul": 54.67625899280575,
        "score": 36.04109399931846
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 51.49058721733788
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 68.24190669801142,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 4.725720094034247
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 68.24190669801142
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 33.934589289450244,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": 100,
        "score": 32.95130885195729
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 33.934589289450244
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 33.62990228437019,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0,
        "score": 64.41911861781645
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 33.62990228437019
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 32.11300719789304,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 25,
        "score": 4.858574666304816
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 32.11300719789304
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 64.97069379895709,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 22.971317562109007
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 64.97069379895709
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 39.099214004618965,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 60.00000000000001,
        "score": 14.485886652458968
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 39.099214004618965
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 32.83013966262374,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": -1.509581012128769
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 32.83013966262374
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.18530071199275,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.5559021359783
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.18530071199275
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 44.550106052514856,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 52.265036772263194
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 44.550106052514856
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -14.516129032258064,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 69.82411392080512,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80,
        "score": 44.70721268920334
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 69.82411392080512
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 62.10287688730801,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": 39.34339789822337
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 62.10287688730801
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 26.944485862981143,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 37.429202269794494
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 26.944485862981143
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 69.43378739054388,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 30.040492606414237
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 69.43378739054388
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 55.79040730990699,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -32.62877807027902
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 55.79040730990699
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 35.76198975772707,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": -0.6236352465927908
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 35.76198975772707
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7213333333333334,
       "cc": 64.00619988901434,
       "cc_by_type": {
        "choice": 64.00619988901434
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 64.00619988901434
      },
      "noul": {
       "accuracy": 0.7786666666666666,
       "cc": 55.73333333333332,
       "cc_by_type": {
        "noul": 55.73333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 55.73333333333332
      },
      "score": {
       "accuracy": 0.52,
       "cc": 35.63329245619988,
       "cc_by_type": {
        "score": 35.63329245619988
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 35.63329245619988
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 31.602907151164914,
       "cc_by_type": {
        "choice": 36,
        "noul": 50,
        "score": 8.808721453494739
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 31.602907151164914
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 24.65440579911538,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 55.55555555555556,
        "score": -25.69171704019699
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 24.65440579911538
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 58.85440387413508,
       "cc_by_type": {
        "choice": 18.699186991869915,
        "noul": 100,
        "score": 57.86402463053535
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 58.85440387413508
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 21.573024553922433,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": 24.952991790422274
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 21.573024553922433
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 28.246437211964235,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": 33.33333333333333,
        "score": 23.869746418501414
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 28.246437211964235
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": 0.38882349872808203,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 33.33333333333333,
        "score": -69.2737810761428
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0.38882349872808203
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 42.964161901689614,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 71.42857142857142,
        "score": -3.320399448992739
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 42.964161901689614
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 14.800234981182662,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -50,
        "score": 46.665513305917315
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 14.800234981182662
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 34.22439466186014,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 64.33985065224708
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 34.22439466186014
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 35.392748862215676,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 3.7514086137705016
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 35.392748862215676
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 49.47199754097719,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 33.33333333333333,
        "score": 38.338473243086646
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 49.47199754097719
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 30.394104568777696,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 22.365109405257822
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 30.394104568777696
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 56.36973577985058,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 60.62552211882236
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 56.36973577985058
      },
      "model_routing": {
       "accuracy": 0.7972350230414746,
       "cc": 66.80548991484399,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 78.78787878787878,
        "score": 42.66755199561424
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.80548991484399
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 48.12446743220059,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0,
        "score": 55.889717075872404
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 48.12446743220059
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 80.53804875918388,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100,
        "score": 58.32723819955723
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 80.53804875918388
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 40.26737295094906,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50,
        "score": 0.35876909915260047
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 40.26737295094906
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 41.32801314674797,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 19.636213353287392
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 41.32801314674797
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 79.25826168668284,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100,
        "score": 59.12709822730834
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 79.25826168668284
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.2065286202611,
      76.13123675368387
     ],
     "I": [
      46.451591543620204,
      56.656336267839215
     ],
     "capability": [
      57.76581304676275,
      65.61559579468398
     ],
     "composite": null,
     "seed": 46
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 72.86180997007926,
     "capability": 62.443616494634156,
     "intelligence": 52.02542301918905
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7213333333333334,
      "calibration": 74.9718083142479,
      "cc": 64.00619988901434,
      "cc_by_type": {
       "choice": 64.00619988901434
      },
      "cc_tiered": 66.46434739882781,
      "cc_tiered_open": 72.70061629878168,
      "cc_tiered_sealed": 60.22807849887394,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 64.00619988901434
     },
     "noul": {
      "accuracy": 0.7786666666666666,
      "calibration": 69.19561916858285,
      "cc": 55.73333333333332,
      "cc_by_type": {
       "noul": 55.73333333333332
      },
      "cc_tiered": 55.12569812255417,
      "cc_tiered_open": 59.21394064872326,
      "cc_tiered_sealed": 51.03745559638509,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 55.73333333333332
     },
     "score": {
      "accuracy": 0.52,
      "calibration": 74.41800242740706,
      "cc": 35.63329245619988,
      "cc_by_type": {
       "score": 35.63329245619988
      },
      "cc_tiered": 34.48622353618516,
      "cc_tiered_open": 34.98097373905368,
      "cc_tiered_sealed": 33.991473333316634,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.63329245619988
     }
    },
    "raw_on_A": {
     "calibration": 76.02710478723333,
     "capability": 64.44192558101349,
     "intelligence": 52.85674637479366
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.77585829247693,
    "Emphasis on Accuracy 60:20:20": 60.9290206678406,
    "Emphasis on Cost 20:20:60": 71.37478628798412,
    "Emphasis on Speed 20:60:20": 75.77555949678077,
    "Intelligence only": 52.02542301918905
   }
  },
  {
   "adapter_id": "images/metask-jev-4b/spec.json:metask_v15.MetaskJevWarm",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Wayfind (metask-ai)",
   "axes": {
    "calibration": 84.41575352936586,
    "cost": 76.02122363625783,
    "intelligence": 39.166454612935595,
    "speed": 90.45494495649334
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06577096449867435,
      "ece_score": 86.84580710026512,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 86.84580710026512,
      "score_before_time_limit": 86.84580710026512,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2506738544474394,
      "brier": 0.14719486396931378,
      "ece": 0.09558018432686369,
      "n": 371,
      "score": 80.88396313462727,
      "score_before_time_limit": 80.88396313462727,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9728997289972899,
      "interval90_width": 0.6723125564588976,
      "n": 369,
      "nrps": 0.11633460342816504,
      "score": 85.51749035320519,
      "score_before_time_limit": 85.51749035320519,
      "time_limit_failed_share": 0,
      "top_ece": 0.08665779475386558
     }
    },
    "score": 84.41575352936586
   },
   "capability": 61.79110407115073,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,041,090 input + 0 output tokens over 1,624 decisions = 641.065270936 tokens/decision. GPU-time cost USD 0.00629930367879192/1,000; old list-price cost USD 0.0256426108374384/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00629930367879192/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.006299303678791922
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "metask-jev-4b",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 41.89224435585361,
    "I_sealed": 36.440664870017585,
    "base": 39.166454612935595,
    "excess": 2.878191843397784,
    "gap": 5.451579485836028,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 60.50866528870256,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 52.815350739226176,
       "judge": 60.44825313117996,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 37.846790890269155,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 27.321276888589132,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.68821311926259,
       "hard": 13.269014128057655,
       "judge": 46.73482367537818,
       "standard": 21.622014114131783
      }
     },
     "sealed|choice": {
      "cc": 56.24961082249847,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 73.3944276106718,
       "hard": 48.076923076923066,
       "judge": 61.824953445065184,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 22.8393257860911,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 63.99999999999999,
       "hard": 2.9126213592232997,
       "judge": 23.684210526315795,
       "standard": 40.8450704225352
      }
     },
     "sealed|score": {
      "cc": 30.23305800146319,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 50.21133272720072,
       "hard": 17.595550281640328,
       "judge": 37.810328121508064,
       "standard": 34.153030898172844
      }
     }
    }
   },
   "jevbench_score": 39.85244736840569,
   "key": "metask-jev-4b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8776978417266187,
    "decisive_rate": 0.7493261455525606,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.41575352936586,
     "cost": 57.7311327584882,
     "intelligence": 39.166454612935595,
     "speed": 90.78239008886905
    },
    "board_release": "v1.6.43",
    "capability": 61.79110407115073,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 53,
    "capability_rank": 78,
    "composite_ci95": {
     "A": [
      24.35782606760525,
      47.92198520846029
     ],
     "B": [
      21.36935197657226,
      43.913080458446345
     ],
     "C": [
      16.915156991392532,
      33.27915639476408
     ]
    },
    "jevbench_score": 37.34931073616377,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 57,
    "rank": 87,
    "ranks": {
     "A": 87,
     "B": 88,
     "C": 88,
     "capability": 122
    },
    "scores": {
     "A": 37.34931073616377,
     "B": 33.62317034896201,
     "C": 25.937021344558172
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25794456995790827,
     "p50_s_raw": 0.05397228497895412,
     "p95_s_adjusted": 0.3237685059779324,
     "p95_s_raw": 0.08688425298896618
    },
    "views": {
     "Balanced 33:33:33": 34.17198402783528,
     "Emphasis on Accuracy 60:20:20": 29.237864520407634,
     "Emphasis on Cost 20:20:60": 34.662048899004674,
     "Emphasis on Speed 20:60:20": 40.422015539591015,
     "Intelligence only": 24.032711498388675
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 148,
   "ranked": true,
   "ranks": {
    "A": 148,
    "B": 150,
    "C": 150,
    "capability": 152
   },
   "repo": "https://github.com/metask-ai/metask-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 39.85244736840569,
    "B": 35.21617790062786,
    "C": 27.67531067250395
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2660654459963553,
    "p50_s_raw": 0.05803272299817763,
    "p95_s_adjusted": 0.3384675424604211,
    "p95_s_raw": 0.09423377123021055
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-4B merged r16 LoRA, candidate-logit readout",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.04800226228174,
      85.90669927321129
     ],
     "I": [
      32.96914634416673,
      43.538341958330456
     ],
     "capability": [
      56.20112405588935,
      63.92462857658634
     ],
     "composite": null,
     "seed": 66
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.41575352936586,
     "capability": 61.79110407115073,
     "intelligence": 39.166454612935595
    },
    "per_type": {
     "choice": {
      "accuracy": 0.668,
      "calibration": 86.84580710026512,
      "cc": 57.541941778842954,
      "cc_by_type": {
       "choice": 57.541941778842954
      },
      "cc_tiered": 58.379138055600514,
      "cc_tiered_open": 60.50866528870256,
      "cc_tiered_sealed": 56.24961082249847,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.541941778842954
     },
     "noul": {
      "accuracy": 0.6506666666666666,
      "calibration": 80.88396313462727,
      "cc": 30.133333333333322,
      "cc_by_type": {
       "noul": 30.133333333333322
      },
      "cc_tiered": 30.34305833818013,
      "cc_tiered_open": 37.846790890269155,
      "cc_tiered_sealed": 22.8393257860911,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 30.133333333333322
     },
     "score": {
      "accuracy": 0.48,
      "calibration": 85.51749035320519,
      "cc": 29.198392850325618,
      "cc_by_type": {
       "score": 29.198392850325618
      },
      "cc_tiered": 28.77716744502616,
      "cc_tiered_open": 27.321276888589132,
      "cc_tiered_sealed": 30.23305800146319,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 29.198392850325618
     }
    },
    "raw_on_A": {
     "calibration": 83.18850868556241,
     "capability": 62.528725908581364,
     "intelligence": 41.86894313160032
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 37.00759580158449,
    "Emphasis on Accuracy 60:20:20": 30.435036176984095,
    "Emphasis on Cost 20:20:60": 40.34220339614661,
    "Emphasis on Speed 20:60:20": 42.69921735671953,
    "Intelligence only": 24.032711498388675
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": null,
   "axes": {
    "calibration": 80.89039994344657,
    "cost": 69.50479145303603,
    "intelligence": 51.08240263971605,
    "speed": 78.38456523323399
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06046272357834501,
      "ece_score": 87.907455284331,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 322,
      "n_tvd": 0,
      "score": 87.907455284331,
      "score_before_time_limit": 87.907455284331,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.16334625066666603,
      "ece": 0.11029866666666785,
      "n": 375,
      "score": 77.94026666666642,
      "score_before_time_limit": 77.94026666666642,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8826666666666667,
      "interval90_width": 0.5564444444444444,
      "n": 375,
      "nrps": 0.1216478762698887,
      "score": 76.82347787934233,
      "score_before_time_limit": 76.82347787934233,
      "time_limit_failed_share": 0,
      "top_ece": 0.1709412830716323
     }
    },
    "score": 80.89039994344657
   },
   "capability": 65.9864012915813,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,456,885 input + 106,568 output tokens over 1,479 decisions = 1057.10141988 tokens/decision. GPU-time cost USD 0.0103874022895879/1,000; old list-price cost USD 0.0403595334685598/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0103874022895879/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.04652426
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.010387402289587948
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Metask rain 4B",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-09",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 56.1234361269253,
    "I_sealed": 46.041369152506796,
    "base": 51.08240263971605,
    "excess": 2.993969230169018,
    "gap": 10.082066974418503,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 73.60321904242878,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 66.02705253224286,
       "judge": 76.26895187870798,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 54.94202898550725,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 39.825060352839884,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 41.599209143896296,
       "hard": 26.675379512412224,
       "judge": 65.98556402232718,
       "standard": 25.996592133936048
      }
     },
     "sealed|choice": {
      "cc": 58.676637540616824,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 68.51527934972734,
       "hard": 52.89814293753516,
       "judge": 62.64236902050115,
       "standard": 59.36570862239843
      }
     },
     "sealed|noul": {
      "cc": 56.6951598827452,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 66.66666666666667,
       "hard": 48.27586206896552,
       "judge": 49.25373134328359,
       "standard": 79.71014492753623
      }
     },
     "sealed|score": {
      "cc": 22.752310034158377,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 28.051523454106352,
       "hard": 29.62563522191024,
       "judge": 13.23325264200742,
       "standard": 20.63463903690709
      }
     }
    }
   },
   "jevbench_score": 67.70053221613982,
   "key": "metask-jev-rain-4b",
   "last_measured_on": "2026-10-09",
   "listing": "ranked",
   "measured_in": "v1.6.2",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "ea20fe85b28733b1522721dec119bec50947a869",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 80.89039994344657,
     "cost": 51.82161587751452,
     "intelligence": 51.08240263971605,
     "speed": 79.59790373470649
    },
    "board_release": "v1.6.43",
    "capability": 65.9864012915813,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 35,
    "capability_rank": 56,
    "composite_ci95": {
     "A": [
      47.26843551984421,
      64.16817644128109
     ],
     "B": [
      44.254055214557084,
      62.11354987809795
     ],
     "C": [
      32.82530244433626,
      54.26142073872429
     ]
    },
    "jevbench_score": 62.69717160365172,
    "measured_in": "v1.6.2",
    "method": "v1.6",
    "open_board_rank": 31,
    "rank": 52,
    "ranks": {
     "A": 52,
     "B": 52,
     "C": 57,
     "capability": 98
    },
    "scores": {
     "A": 62.69717160365172,
     "B": 59.97005503140675,
     "C": 45.44520642674837
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26540608701761814,
     "p50_s_raw": 0.05770304350880906,
     "p95_s_adjusted": 4.133317315817111,
     "p95_s_raw": 1.9916586579085556
    },
    "views": {
     "Balanced 33:33:33": 58.3245403206291,
     "Emphasis on Accuracy 60:20:20": 55.194490971444196,
     "Emphasis on Cost 20:20:60": 55.53688383211001,
     "Emphasis on Speed 20:60:20": 65.30602465884293,
     "Intelligence only": 51.08240263971605
    }
   },
   "rank": 97,
   "ranked": true,
   "ranks": {
    "A": 97,
    "B": 100,
    "C": 104,
    "capability": 127
   },
   "repo": "https://huggingface.co/wayfind/metask-jev-4b-policy-mix",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.70053221613982,
    "B": 63.564754447970955,
    "C": 49.07182545989054
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3165951920440421,
    "p50_s_raw": 0.08329759602202103,
    "p95_s_adjusted": 4.581832004513126,
    "p95_s_raw": 2.215916002256563
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.96,
       "cc": 94.84452702447224,
       "cc_by_type": {
        "choice": 91.1111111111111,
        "noul": 100,
        "score": 93.42246996230561
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 94.84452702447224
      },
      "calibration_gen": {
       "accuracy": 0.5520833333333334,
       "cc": 26.715869992998762,
       "cc_by_type": {
        "choice": 46.50205761316872,
        "noul": 58.33333333333333,
        "score": -24.687780967505766
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 26.715869992998762
      },
      "composite_gen": {
       "accuracy": 0.59,
       "cc": 30.609363116297118,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 60.00000000000001,
        "score": -19.39142284623059
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 30.609363116297118
      },
      "fairness": {
       "accuracy": 0.9761904761904762,
       "cc": 96.75272495371574,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 90.25817486114721
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 96.75272495371574
      },
      "jaggedness": {
       "accuracy": 0.5432098765432098,
       "cc": 24.77815252618288,
       "cc_by_type": {
        "choice": 42.06486754147067,
        "noul": 42.85714285714286,
        "score": -10.587552820064893
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 24.77815252618288
      },
      "llm_router": {
       "accuracy": 0.8084415584415584,
       "cc": 69.36316339410013,
       "cc_by_type": {
        "choice": 75.92080620708107,
        "noul": 85.41666666666667,
        "score": 46.75201730855263
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 69.36316339410013
      },
      "long_state": {
       "accuracy": 0.38095238095238093,
       "cc": -7.847225627137181,
       "cc_by_type": {
        "choice": 22.413793103448274,
        "noul": 0,
        "score": -45.95546998485982
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5882352941176471,
       "cc": 41.08191998802071,
       "cc_by_type": {
        "choice": 47.36842105263158,
        "noul": 50,
        "score": 25.877338911430535
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 41.08191998802071
      },
      "multilingual": {
       "accuracy": 0.6701754385964912,
       "cc": 51.06107828522669,
       "cc_by_type": {
        "choice": 57.97011207970113,
        "noul": 46.34146341463414,
        "score": 48.8716593613448
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 51.06107828522669
      },
      "negation_pair_gen": {
       "accuracy": 0.6428571428571429,
       "cc": 36.463286789996,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 41.176470588235304,
        "score": 1.546723115086024
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 36.463286789996
      },
      "option_order": {
       "accuracy": 0.7619047619047619,
       "cc": 69.66632962588473,
       "cc_by_type": {
        "choice": 69.66632962588473
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 69.66632962588473
      },
      "perturb_pair": {
       "accuracy": 0.7741935483870968,
       "cc": 65.19271120746511,
       "cc_by_type": {
        "choice": 71.69811320754718,
        "noul": 66.66666666666667,
        "score": 57.21335374818144
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 65.19271120746511
      },
      "usecase": {
       "accuracy": 0.5686274509803921,
       "cc": 40.31421175143449,
       "cc_by_type": {
        "choice": 53.696788648244976,
        "noul": 39.99999999999999,
        "score": 27.2458466060585
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 40.31421175143449
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4117647058823529,
       "cc": -0.07450502189792256,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0,
        "score": -43.08065792283662
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5555555555555556,
       "cc": 35.632183908045974,
       "cc_by_type": {
        "choice": 37.93103448275862,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 35.632183908045974
      },
      "da": {
       "accuracy": 0.6875,
       "cc": 60.90270576587461,
       "cc_by_type": {
        "choice": 53.327571305099404,
        "noul": 100,
        "score": 29.380545992524432
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 60.90270576587461
      },
      "de": {
       "accuracy": 0.775,
       "cc": 62.41736197307345,
       "cc_by_type": {
        "choice": 70.3581720872787,
        "noul": 81.81818181818181,
        "score": 35.07573201375983
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 62.41736197307345
      },
      "el": {
       "accuracy": 0.7647058823529411,
       "cc": 73.37747350577037,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 60.00000000000001,
        "score": 79.18003956493014
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 73.37747350577037
      },
      "en": {
       "accuracy": 0.6713656387665198,
       "cc": 48.42563024796656,
       "cc_by_type": {
        "choice": 62.24682287529783,
        "noul": 61.19402985074627,
        "score": 21.836038017855586
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 48.42563024796656
      },
      "es": {
       "accuracy": 0.7666666666666667,
       "cc": 65.47988597079195,
       "cc_by_type": {
        "choice": 63.879598662207364,
        "noul": 81.81818181818181,
        "score": 50.74187743198667
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 65.47988597079195
      },
      "fi": {
       "accuracy": 0,
       "cc": -24.70996081033493,
       "cc_by_type": {
        "choice": -28.205128205128208,
        "noul": -100,
        "score": 54.075245774123424
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.625,
       "cc": 29.6779881965786,
       "cc_by_type": {
        "choice": 56.62650602409638,
        "noul": -33.333333333333336,
        "score": 65.74079189897277
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 29.6779881965786
      },
      "hi": {
       "accuracy": 0.6153846153846154,
       "cc": 40.80657497403471,
       "cc_by_type": {
        "choice": 44.547134935304996,
        "noul": 66.66666666666667,
        "score": 11.205923320132472
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 40.80657497403471
      },
      "id": {
       "accuracy": 0.5,
       "cc": 34.42622950819673,
       "cc_by_type": {
        "choice": 34.42622950819673
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 34.42622950819673
      },
      "it": {
       "accuracy": 0.9230769230769231,
       "cc": 92.98419698867792,
       "cc_by_type": {
        "choice": 83.98576512455516,
        "noul": 100,
        "score": 94.96682584147862
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 92.98419698867792
      },
      "ja": {
       "accuracy": 0.45454545454545453,
       "cc": 7.879051793920785,
       "cc_by_type": {
        "choice": 43.172849250197324,
        "noul": 0,
        "score": -19.53569386843497
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 7.879051793920785
      },
      "ko": {
       "accuracy": 0.6666666666666666,
       "cc": 64.53133627110843,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 100,
        "score": 20.260675479991942
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 64.53133627110843
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.8719181249276,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.61575437478281
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 99.8719181249276
      },
      "nl": {
       "accuracy": 0.4117647058823529,
       "cc": -15.949562639532397,
       "cc_by_type": {
        "choice": 28.115015974440897,
        "noul": -50,
        "score": -25.96370389303808
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.75,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 60.219275506836446,
       "cc_by_type": {
        "choice": 84.92462311557787,
        "noul": 66.66666666666667,
        "score": 29.0665367382648
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 60.219275506836446
      },
      "pt": {
       "accuracy": 0.8333333333333334,
       "cc": 74.11075635925,
       "cc_by_type": {
        "choice": 87.30606488011284,
        "noul": 75,
        "score": 60.026204197637156
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 74.11075635925
      },
      "sv": {
       "accuracy": 0.5,
       "cc": 29.78604643849067,
       "cc_by_type": {
        "choice": 68.9119170984456,
        "noul": -33.333333333333336,
        "score": 53.77955555035974
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 29.78604643849067
      },
      "tr": {
       "accuracy": 0.5454545454545454,
       "cc": 5.547657907361341,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": -39.87876540835076
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 5.547657907361341
      },
      "uk": {
       "accuracy": 0.8888888888888888,
       "cc": 80.82177970414698,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 74.72340362856995
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 80.82177970414698
      },
      "zh": {
       "accuracy": 0.6666666666666666,
       "cc": 52.23199799021912,
       "cc_by_type": {
        "choice": 36.170212765957444,
        "noul": 66.66666666666667,
        "score": 53.859114538033225
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 52.23199799021912
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.704,
       "cc": 61.67935710312817,
       "cc_by_type": {
        "choice": 61.67935710312817
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 61.67935710312817
      },
      "noul": {
       "accuracy": 0.7946666666666666,
       "cc": 58.93333333333332,
       "cc_by_type": {
        "noul": 58.93333333333332
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 58.93333333333332
      },
      "score": {
       "accuracy": 0.48533333333333334,
       "cc": 26.7077945049382,
       "cc_by_type": {
        "score": 26.7077945049382
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 26.7077945049382
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5666666666666667,
       "cc": 28.124915716071694,
       "cc_by_type": {
        "choice": 36.39575971731449,
        "noul": 50,
        "score": -2.021012569099412
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 28.124915716071694
      },
      "code_linting": {
       "accuracy": 0.4090909090909091,
       "cc": 28.254375685619937,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 60.00000000000001,
        "score": -0.70271145245695
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 28.254375685619937
      },
      "customer_support": {
       "accuracy": 0.6923076923076923,
       "cc": 59.49337631484335,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 88.62505648076197
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 59.49337631484335
      },
      "demand_forecasting": {
       "accuracy": 0.5238095238095238,
       "cc": 21.86363398640653,
       "cc_by_type": {
        "choice": 61.03896103896104,
        "noul": 0,
        "score": 4.5519409202585575
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 21.86363398640653
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7142857142857143,
       "cc": 53.51614932057469,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 60.00000000000001,
        "score": 33.335333207625695
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 53.51614932057469
      },
      "feature_extraction": {
       "accuracy": 0.5625,
       "cc": 5.923903534888588,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -33.333333333333336,
        "score": 13.605043937999096
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 5.923903534888588
      },
      "financial_crime": {
       "accuracy": 0.6071428571428571,
       "cc": 48.94759929822106,
       "cc_by_type": {
        "choice": 78.72340425531915,
        "noul": 60.00000000000001,
        "score": 8.119393639343997
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 48.94759929822106
      },
      "gaming": {
       "accuracy": 0.35294117647058826,
       "cc": 24.14019231524544,
       "cc_by_type": {
        "choice": 43.79391100702576,
        "noul": 0,
        "score": 28.62666593871055
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 24.14019231524544
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 25.272604681173657,
       "cc_by_type": {
        "choice": 21.052631578947366,
        "noul": 0,
        "score": 54.765182464573606
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 25.272604681173657
      },
      "knowledge_graphs": {
       "accuracy": 0.42857142857142855,
       "cc": 18.187989919020836,
       "cc_by_type": {
        "choice": 20.353982300884958,
        "noul": 14.28571428571428,
        "score": 19.92427317046327
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 18.187989919020836
      },
      "lead_generation": {
       "accuracy": 0.6296296296296297,
       "cc": 55.683058307434294,
       "cc_by_type": {
        "choice": 49.4949494949495,
        "noul": 80,
        "score": 37.55422542735337
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 55.683058307434294
      },
      "legal_compliance": {
       "accuracy": 0.7333333333333333,
       "cc": 49.52547609201625,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 50,
        "score": 24.892217749732946
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 49.52547609201625
      },
      "llm_guardrails": {
       "accuracy": 0.75,
       "cc": 46.187284107298744,
       "cc_by_type": {
        "choice": 50.81967213114754,
        "noul": 14.28571428571428,
        "score": 73.4564659050344
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 46.187284107298744
      },
      "model_routing": {
       "accuracy": 0.7783251231527094,
       "cc": 63.54604802056192,
       "cc_by_type": {
        "choice": 77.35549228630074,
        "noul": 83.05084745762711,
        "score": 30.231804317757916
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 63.54604802056192
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 19.915539893895183,
       "cc_by_type": {
        "choice": 31.81818181818182,
        "noul": 0,
        "score": 27.92843786350373
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 19.915539893895183
      },
      "recruiting": {
       "accuracy": 0.5882352941176471,
       "cc": 67.38856406151355,
       "cc_by_type": {
        "choice": 43.573667711598745,
        "noul": 100,
        "score": 58.592024472941894
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 67.38856406151355
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 25.997491018065205,
       "cc_by_type": {
        "choice": 70.6601466992665,
        "noul": -33.333333333333336,
        "score": 40.665659688262444
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 25.997491018065205
      },
      "scientific_discovery": {
       "accuracy": 0.38095238095238093,
       "cc": 17.543893040423583,
       "cc_by_type": {
        "choice": 12.621359223300967,
        "noul": 0,
        "score": 40.01031989796978
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 17.543893040423583
      },
      "search_retrieval": {
       "accuracy": 0.75,
       "cc": 70.40470952339105,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 11.214128570173177
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 70.40470952339105
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.5497718269952,
      83.39564578949536
     ],
     "I": [
      44.504765007776726,
      55.16844250289851
     ],
     "capability": [
      61.09493578058464,
      68.36932275256177
     ],
     "composite": null,
     "seed": 18
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 80.89039994344657,
     "capability": 65.9864012915813,
     "intelligence": 51.08240263971605
    },
    "per_type": {
     "choice": {
      "accuracy": 0.704,
      "calibration": 87.907455284331,
      "cc": 61.67935710312817,
      "cc_by_type": {
       "choice": 61.67935710312817
      },
      "cc_tiered": 66.1399282915228,
      "cc_tiered_open": 73.60321904242878,
      "cc_tiered_sealed": 58.676637540616824,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.67935710312817
     },
     "noul": {
      "accuracy": 0.7946666666666666,
      "calibration": 77.94026666666642,
      "cc": 58.93333333333332,
      "cc_by_type": {
       "noul": 58.93333333333332
      },
      "cc_tiered": 55.818594434126226,
      "cc_tiered_open": 54.94202898550725,
      "cc_tiered_sealed": 56.6951598827452,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 58.93333333333332
     },
     "score": {
      "accuracy": 0.48533333333333334,
      "calibration": 76.82347787934233,
      "cc": 26.7077945049382,
      "cc_by_type": {
       "score": 26.7077945049382
      },
      "cc_tiered": 31.28868519349913,
      "cc_tiered_open": 39.825060352839884,
      "cc_tiered_sealed": 22.752310034158377,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 26.7077945049382
     }
    },
    "raw_on_A": {
     "calibration": 77.62323799437446,
     "capability": 66.62869754737606,
     "intelligence": 55.63415710037767
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.21050900345246,
    "Emphasis on Accuracy 60:20:20": 58.22501036159797,
    "Emphasis on Cost 20:20:60": 66.22839479757242,
    "Emphasis on Speed 20:60:20": 69.21703630497579,
    "Intelligence only": 51.08240263971605
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": null,
   "axes": {
    "calibration": 89.84415618532353,
    "cost": 65.2582052479506,
    "intelligence": 66.74877414880126,
    "speed": 75.19061343032087
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06831102916513103,
      "ece_score": 86.3377941669738,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 322,
      "n_tvd": 0,
      "score": 86.3377941669738,
      "score_before_time_limit": 86.3377941669738,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.12734986666666614,
      "ece": 0.0327466666666661,
      "n": 375,
      "score": 93.45066666666678,
      "score_before_time_limit": 93.45066666666678,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9306666666666666,
      "interval90_width": 0.3253333333333333,
      "n": 375,
      "nrps": 0.07579799787223554,
      "score": 89.74400772233005,
      "score_before_time_limit": 89.74400772233005,
      "time_limit_failed_share": 0,
      "top_ece": 0.06466092384058167
     }
    },
    "score": 89.84415618532353
   },
   "capability": 78.2964651670624,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 859,455 input + 86,179 output tokens over 1,479 decisions = 639.373901285 tokens/decision. GPU-time cost USD 0.0143899678973605/1,000; old list-price cost USD 0.037795537525355/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0143899678973605/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.04279556666666667
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014389967897360545
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "metask-jev-rain-12B",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-09",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 70.645105918311,
    "I_sealed": 62.85244237929151,
    "base": 66.74877414880126,
    "excess": 0.7045657947700086,
    "gap": 7.792663539019493,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 77.71876710209611,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 69.80182447310474,
       "judge": 72.31377719182598,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 82.1207729468599,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 89.99999999999999,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 52.09577770597702,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 76.55742918641428,
       "hard": 53.19893501712373,
       "judge": 41.32419472837796,
       "standard": 53.816011809863554
      }
     },
     "sealed|choice": {
      "cc": 72.26629481435496,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 96.29591521761498,
       "hard": 61.508159819921204,
       "judge": 72.66514806378133,
       "standard": 81.16947472745294
      }
     },
     "sealed|noul": {
      "cc": 64.72683434402202,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 75,
       "hard": 60.3448275862069,
       "judge": 55.223880597014926,
       "standard": 82.6086956521739
      }
     },
     "sealed|score": {
      "cc": 51.56419797949755,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 76.8746607073365,
       "hard": 42.46762350094215,
       "judge": 55.43267392491627,
       "standard": 51.299401654560796
      }
     }
    }
   },
   "jevbench_score": 73.07910352308468,
   "key": "metask_jev_rain_12b",
   "last_measured_on": "2026-10-09",
   "listing": "wrapper",
   "measured_in": "v1.6.2",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "12deda07c573047b191b63f0768c0412120f3ed4",
   "not_ranked_because": "registry ranked=false or no composite",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.84415618532353,
     "cost": 52.6767842132323,
     "intelligence": 66.74877414880126,
     "speed": 75.9887043236542
    },
    "board_release": "v1.6.43",
    "capability": 78.2964651670624,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      66.9519161575783,
      69.51190178423195
     ],
     "B": [
      65.86465215953126,
      69.73439376375767
     ],
     "C": [
      66.9519161575783,
      69.51190178423195
     ]
    },
    "jevbench_score": 68.66306996476943,
    "measured_in": "v1.6.2",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": 68.66306996476943,
     "B": 68.27147684820034,
     "C": 68.66306996476943
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.33421517265960576,
     "p50_s_raw": 0.09210758632980287,
     "p95_s_adjusted": 7.53534959163516,
     "p95_s_raw": 3.6926747958175796
    },
    "views": {
     "Balanced 33:33:33": 63.66035034197887,
     "Emphasis on Accuracy 60:20:20": 64.86077729380042,
     "Emphasis on Cost 20:20:60": 58.75959627350012,
     "Emphasis on Speed 20:60:20": 68.07834337436685,
     "Intelligence only": 66.74877414880126
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/cyankiwi/gemma-4-12B-it-qat-AWQ-INT4",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.07910352308468,
    "B": 71.71876864464473,
    "C": 73.07910352308468
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.35249755401164296,
    "p50_s_raw": 0.10124877700582147,
    "p95_s_adjusted": 8.585835219966249,
    "p95_s_raw": 4.217917609983124
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 96.27347996861447,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 88.82043990584336
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 96.27347996861447
      },
      "calibration_gen": {
       "accuracy": 0.7708333333333334,
       "cc": 57.06717856711694,
       "cc_by_type": {
        "choice": 58.8477366255144,
        "noul": 66.66666666666667,
        "score": 45.68713240916975
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 57.06717856711694
      },
      "composite_gen": {
       "accuracy": 0.79,
       "cc": 66.4219048681923,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 68,
        "score": 47.52587720620293
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 66.4219048681923
      },
      "fairness": {
       "accuracy": 0.9761904761904762,
       "cc": 93.12691462021985,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 79.38074386065956
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 93.12691462021985
      },
      "jaggedness": {
       "accuracy": 0.6419753086419753,
       "cc": 39.87106908082783,
       "cc_by_type": {
        "choice": 59.891062144095095,
        "noul": 21.42857142857142,
        "score": 38.29357366981698
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 39.87106908082783
      },
      "llm_router": {
       "accuracy": 0.8506493506493507,
       "cc": 77.84045308249219,
       "cc_by_type": {
        "choice": 75.11816641398377,
        "noul": 91.66666666666667,
        "score": 66.73652616682611
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 77.84045308249219
      },
      "long_state": {
       "accuracy": 0.14285714285714285,
       "cc": -36.80944920727243,
       "cc_by_type": {
        "choice": -12.068965517241384,
        "noul": 0,
        "score": -98.35938210457591
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6764705882352942,
       "cc": 54.42227412247834,
       "cc_by_type": {
        "choice": 47.36842105263158,
        "noul": 75,
        "score": 40.898401314803436
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 54.42227412247834
      },
      "multilingual": {
       "accuracy": 0.8140350877192982,
       "cc": 70.31326141551826,
       "cc_by_type": {
        "choice": 80.38605230386052,
        "noul": 68.29268292682926,
        "score": 62.26104901586497
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 70.31326141551826
      },
      "negation_pair_gen": {
       "accuracy": 0.7959183673469388,
       "cc": 66.23454638878256,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 52.941176470588225,
        "score": 62.429129362426124
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 66.23454638878256
      },
      "option_order": {
       "accuracy": 0.9206349206349206,
       "cc": 89.88877654196155,
       "cc_by_type": {
        "choice": 89.88877654196155
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 89.88877654196155
      },
      "perturb_pair": {
       "accuracy": 0.9193548387096774,
       "cc": 85.30050438932237,
       "cc_by_type": {
        "choice": 87.42138364779876,
        "noul": 100,
        "score": 68.48012952016833
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 85.30050438932237
      },
      "usecase": {
       "accuracy": 0.6862745098039216,
       "cc": 54.519610793076986,
       "cc_by_type": {
        "choice": 61.16504854368934,
        "noul": 60.00000000000001,
        "score": 42.3937838355416
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 54.519610793076986
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.7058823529411765,
       "cc": 47.2113436309134,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100,
        "score": -1.2231119644026789
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 47.2113436309134
      },
      "cs": {
       "accuracy": 0.7777777777777778,
       "cc": 79.3103448275862,
       "cc_by_type": {
        "choice": 58.62068965517241,
        "noul": 100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 79.3103448275862
      },
      "da": {
       "accuracy": 0.75,
       "cc": 66.81661442841975,
       "cc_by_type": {
        "choice": 68.88504753673294,
        "noul": 100,
        "score": 31.56479574852632
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 66.81661442841975
      },
      "de": {
       "accuracy": 0.9,
       "cc": 82.64841002900086,
       "cc_by_type": {
        "choice": 85.17908604363934,
        "noul": 81.81818181818181,
        "score": 80.9479622251814
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 82.64841002900086
      },
      "el": {
       "accuracy": 0.8235294117647058,
       "cc": 71.27641118431332,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 60.00000000000001,
        "score": 91.92447164817807
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 71.27641118431332
      },
      "en": {
       "accuracy": 0.7779735682819383,
       "cc": 64.42039924274495,
       "cc_by_type": {
        "choice": 72.52283558379663,
        "noul": 68.65671641791045,
        "score": 52.08164572652778
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 64.42039924274495
      },
      "es": {
       "accuracy": 0.8666666666666667,
       "cc": 78.93980801601477,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 63.63636363636365,
        "score": 73.18306041168066
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 78.93980801601477
      },
      "fi": {
       "accuracy": 0.7142857142857143,
       "cc": 69.69274830053051,
       "cc_by_type": {
        "choice": 48.71794871794871,
        "noul": 100,
        "score": 60.360296183642824
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 69.69274830053051
      },
      "fr": {
       "accuracy": 0.875,
       "cc": 70.65012917104805,
       "cc_by_type": {
        "choice": 85.5421686746988,
        "noul": 33.33333333333333,
        "score": 93.07488550511206
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 70.65012917104805
      },
      "hi": {
       "accuracy": 0.7307692307692307,
       "cc": 65.13808631336717,
       "cc_by_type": {
        "choice": 77.818853974122,
        "noul": 100,
        "score": 17.595404965979476
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 65.13808631336717
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.9230769230769231,
       "cc": 85.47109020402759,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 56.41327061208279
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 85.47109020402759
      },
      "ja": {
       "accuracy": 0.6818181818181818,
       "cc": 41.64421198154423,
       "cc_by_type": {
        "choice": 57.37963693764798,
        "noul": 50,
        "score": 17.552999006984706
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 41.64421198154423
      },
      "ko": {
       "accuracy": 0.8888888888888888,
       "cc": 90.66541975155566,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 100,
        "score": 98.66292592133367
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 90.66541975155566
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.45235001995381,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.35705005986141
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 99.45235001995381
      },
      "nl": {
       "accuracy": 0.8823529411764706,
       "cc": 67.00050248305712,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 1.0015074491713571
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 67.00050248305712
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 54.68551851777729,
       "cc_by_type": {
        "choice": 84.92462311557787,
        "noul": 50,
        "score": 29.13193243775398
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 54.68551851777729
      },
      "pt": {
       "accuracy": 0.875,
       "cc": 77.86081066893884,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 100,
        "score": 58.970302246590855
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 77.86081066893884
      },
      "sv": {
       "accuracy": 0.375,
       "cc": 31.056095408809714,
       "cc_by_type": {
        "choice": 6.7357512953367875,
        "noul": 33.33333333333333,
        "score": 53.09920159775903
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 31.056095408809714
      },
      "tr": {
       "accuracy": 0.8181818181818182,
       "cc": 40.62563994405945,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 21.876919832178356
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 40.62563994405945
      },
      "uk": {
       "accuracy": 0.7777777777777778,
       "cc": 64.70150517724299,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 60.771182198395636
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 64.70150517724299
      },
      "zh": {
       "accuracy": 0.75,
       "cc": 13.841206507530577,
       "cc_by_type": {
        "choice": 68.08510638297872,
        "noul": 100,
        "score": -126.56148686038699
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 13.841206507530577
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7973333333333333,
       "cc": 73.76244270124091,
       "cc_by_type": {
        "choice": 73.76244270124091
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 73.76244270124091
      },
      "noul": {
       "accuracy": 0.8506666666666667,
       "cc": 70.13333333333334,
       "cc_by_type": {
        "noul": 70.13333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 70.13333333333334
      },
      "score": {
       "accuracy": 0.6906666666666667,
       "cc": 52.99097069055308,
       "cc_by_type": {
        "score": 52.99097069055308
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 52.99097069055308
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6666666666666666,
       "cc": 44.74692355042208,
       "cc_by_type": {
        "choice": 46.996466431095406,
        "noul": 50,
        "score": 37.24430422017083
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 44.74692355042208
      },
      "code_linting": {
       "accuracy": 0.6363636363636364,
       "cc": 39.53589630861794,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 19.999999999999996,
        "score": 17.24122929852465
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 39.53589630861794
      },
      "customer_support": {
       "accuracy": 0.7692307692307693,
       "cc": 55.53391206201687,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 66.66666666666667,
        "score": 43.41333038894915
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 55.53391206201687
      },
      "demand_forecasting": {
       "accuracy": 0.38095238095238093,
       "cc": 13.954116461127205,
       "cc_by_type": {
        "choice": 9.090909090909092,
        "noul": 50,
        "score": -17.22855970752748
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 13.954116461127205
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7142857142857143,
       "cc": 45.76413173276987,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 19.999999999999996,
        "score": 50.07928044421125
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 45.76413173276987
      },
      "feature_extraction": {
       "accuracy": 0.9375,
       "cc": 80.19349160347362,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 40.580474810420874
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 80.19349160347362
      },
      "financial_crime": {
       "accuracy": 0.7857142857142857,
       "cc": 79.86221446859615,
       "cc_by_type": {
        "choice": 78.72340425531915,
        "noul": 100,
        "score": 60.86323915046927
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 79.86221446859615
      },
      "gaming": {
       "accuracy": 0.7058823529411765,
       "cc": 73.12003051112781,
       "cc_by_type": {
        "choice": 71.89695550351288,
        "noul": 100,
        "score": 47.46313602987057
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 73.12003051112781
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 59.563018564812516,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100,
        "score": 41.84695043127965
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 59.563018564812516
      },
      "knowledge_graphs": {
       "accuracy": 0.5714285714285714,
       "cc": 33.961660837751445,
       "cc_by_type": {
        "choice": 46.902654867256636,
        "noul": 42.85714285714286,
        "score": 12.125184788854849
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 33.961660837751445
      },
      "lead_generation": {
       "accuracy": 0.8888888888888888,
       "cc": 69.43920550900503,
       "cc_by_type": {
        "choice": 100,
        "noul": 80,
        "score": 28.31761652701511
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 69.43920550900503
      },
      "legal_compliance": {
       "accuracy": 0.8,
       "cc": 65.65653183952269,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 46.96959551856804
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 65.65653183952269
      },
      "llm_guardrails": {
       "accuracy": 0.7916666666666666,
       "cc": 61.810127083374105,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 71.42857142857142,
        "score": 46.78869506745254
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 61.810127083374105
      },
      "model_routing": {
       "accuracy": 0.8817733990147784,
       "cc": 80.52579213285817,
       "cc_by_type": {
        "choice": 85.69820565450574,
        "noul": 100,
        "score": 55.87917074406879
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 80.52579213285817
      },
      "moderation_trust_safety": {
       "accuracy": 0.7083333333333334,
       "cc": 72.50641191707528,
       "cc_by_type": {
        "choice": 61.038961038961034,
        "noul": 100,
        "score": 56.4802747122648
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 72.50641191707528
      },
      "recruiting": {
       "accuracy": 0.7647058823529411,
       "cc": 65.2415200092604,
       "cc_by_type": {
        "choice": 62.382445141065844,
        "noul": 50,
        "score": 83.34211488671534
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 65.2415200092604
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 47.00039821586588,
       "cc_by_type": {
        "choice": 70.6601466992665,
        "noul": 33.33333333333333,
        "score": 37.0077146149978
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 47.00039821586588
      },
      "scientific_discovery": {
       "accuracy": 0.6190476190476191,
       "cc": 38.963742930892586,
       "cc_by_type": {
        "choice": 27.18446601941747,
        "noul": 50,
        "score": 39.70676277326029
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 38.963742930892586
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 84.49544603640166,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 53.486338109204986
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 84.49544603640166
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.9726432272681,
      90.86115546395659
     ],
     "I": [
      61.92201793536035,
      70.62816650076428
     ],
     "capability": [
      74.31683118509996,
      79.90344483944872
     ],
     "composite": null,
     "seed": 19
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.84415618532353,
     "capability": 78.2964651670624,
     "intelligence": 66.74877414880126
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7973333333333333,
      "calibration": 86.3377941669738,
      "cc": 73.76244270124091,
      "cc_by_type": {
       "choice": 73.76244270124091
      },
      "cc_tiered": 74.99253095822553,
      "cc_tiered_open": 77.71876710209611,
      "cc_tiered_sealed": 72.26629481435496,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.76244270124091
     },
     "noul": {
      "accuracy": 0.8506666666666667,
      "calibration": 93.45066666666678,
      "cc": 70.13333333333334,
      "cc_by_type": {
       "noul": 70.13333333333334
      },
      "cc_tiered": 73.42380364544096,
      "cc_tiered_open": 82.1207729468599,
      "cc_tiered_sealed": 64.72683434402202,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 70.13333333333334
     },
     "score": {
      "accuracy": 0.6906666666666667,
      "calibration": 89.74400772233005,
      "cc": 52.99097069055308,
      "cc_by_type": {
       "score": 52.99097069055308
      },
      "cc_tiered": 51.82998784273728,
      "cc_tiered_open": 52.09577770597702,
      "cc_tiered_sealed": 51.56419797949755,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.99097069055308
     }
    },
    "raw_on_A": {
     "calibration": 85.98840560301178,
     "capability": 77.77430900473493,
     "intelligence": 69.56021240645808
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.79972655237066,
    "Emphasis on Accuracy 60:20:20": 67.96440478872088,
    "Emphasis on Cost 20:20:60": 67.33797100452597,
    "Emphasis on Speed 20:60:20": 71.22112536679197,
    "Intelligence only": 66.74877414880126
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "MeTask (Hangzhou MeTask Technology Co., Ltd.)",
   "axes": {
    "calibration": 87.29383335481249,
    "cost": 65.18480974745836,
    "intelligence": 66.48910229704197,
    "speed": 71.7899932169731
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04579106842955546,
      "ece_score": 90.8417863140889,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 89.87280726007195,
      "score_before_time_limit": 90.8417863140889,
      "time_limit_failed_share": 0.010666666666666666
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.12338284879474153,
      "ece": 0.05903423423423422,
      "n": 375,
      "score": 88.19315315315316,
      "score_before_time_limit": 88.19315315315316,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9226666666666666,
      "interval90_width": 0.28755555555555545,
      "n": 375,
      "nrps": 0.07999688319975698,
      "score": 83.81553965121238,
      "score_before_time_limit": 84.71921123774837,
      "time_limit_failed_share": 0.010666666666666666,
      "top_ece": 0.11280944602263793
     }
    },
    "score": 87.29383335481249
   },
   "capability": 76.89146782592724,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 870,611 input + 84,223 output tokens over 1,485 decisions = 642.985858586 tokens/decision. GPU-time cost USD 0.0144712598448526/1,000; old list-price cost USD 0.0378208754208754/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0144712598448526/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.041404166666666666
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014471259844852572
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 12,
    "n_over_limit_answered_ok": 12,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "metask-jev-rain-12B-trained",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline read-only container (--network none); author's own service (serve.sh -> launch.py -> server.py on vLLM 0.31.0, pinned digest image), unchanged defaults paid evaluation (public). Own LoRA adapter (rank 16) on the author's int4 Gemma 4 12B base, applied unmerged by vLLM. Replaces nothing: the earlier metask_jev_rain_12b row (third-party AWQ wrapper, paid evaluation) is a different system.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 71.19688579914401,
    "I_sealed": 61.781318794939914,
    "base": 66.48910229704197,
    "excess": -0.42395976615161146,
    "gap": 9.4155670042041,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 79.70868696671687,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 77.35136835482857,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 79.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 70,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 54.28293661429005,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 76.59298352366106,
       "hard": 53.42979629970057,
       "judge": 51.83502724515816,
       "standard": 48.50605784248132
      }
     },
     "sealed|choice": {
      "cc": 68.44910042331313,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 84.75222363405337,
       "hard": 64.09159245981265,
       "judge": 62.97747306562194,
       "standard": 77.2199957814807
      }
     },
     "sealed|noul": {
      "cc": 66.55391237686658,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 52.727272727272734,
       "judge": 69.23076923076923,
       "standard": 82.08955223880596
      }
     },
     "sealed|score": {
      "cc": 50.340943584640044,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 62.57017720623247,
       "hard": 48.706025420365464,
       "judge": 47.129277578702755,
       "standard": 52.31366211129891
      }
     }
    }
   },
   "jevbench_score": 71.72811700838548,
   "key": "metask_jev_rain_12b_trained",
   "last_measured_on": "2026-10-12",
   "licence": "Weights: Apache-2.0 per model card (base Gemma 4 under the Gemma 4 licence); service code MIT (github.com/chenyuwang166-oss/metask-jev-rain-12B-trained@e36c7a54)",
   "listing": "ranked",
   "measured_in": "v1.6.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "MeTask-Rain/metask-jev-rain-12B-trained@cf3706fdd57e23025a8c97fbde86bd5b4a547010",
   "noul_decisive": {
    "acc_among_decisive": 0.856,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.91805023499681,
     "cost": 52.66805268932843,
     "intelligence": 67.8255189476327,
     "speed": 73.37579625429389
    },
    "board_release": "v1.6.43",
    "capability": 77.87178459131476,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 9,
    "capability_rank": 18,
    "composite_ci95": {
     "A": [
      66.36559898881377,
      68.99113382462001
     ],
     "B": [
      65.70425913953832,
      69.45086045975164
     ],
     "C": [
      66.36559898881377,
      68.99113382462001
     ]
    },
    "jevbench_score": 68.10451633937213,
    "measured_in": "v1.6.10",
    "method": "v1.6",
    "open_board_rank": 18,
    "rank": 37,
    "ranks": {
     "A": 37,
     "B": 29,
     "C": 31,
     "capability": 34
    },
    "scores": {
     "A": 68.10451633937213,
     "B": 68.0485333889376,
     "C": 68.10451633937213
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3790490360464901,
     "p50_s_raw": 0.11452451802324504,
     "p95_s_adjusted": 12.126207013113888,
     "p95_s_raw": 5.988103506556944
    },
    "views": {
     "Balanced 33:33:33": 63.34589532500726,
     "Emphasis on Accuracy 60:20:20": 65.06481097061045,
     "Emphasis on Cost 20:20:60": 58.59417683415789,
     "Emphasis on Speed 20:60:20": 67.00978182322413,
     "Intelligence only": 67.8255189476327
    }
   },
   "rank": 61,
   "ranked": true,
   "ranks": {
    "A": 61,
    "B": 49,
    "C": 47,
    "capability": 54
   },
   "repo": "https://huggingface.co/MeTask-Rain/metask-jev-rain-12B-trained",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.72811700838548,
    "B": 70.61528941022313,
    "C": 71.72811700838548
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.40733872305136176,
    "p50_s_raw": 0.12866936152568087,
    "p95_s_adjusted": 16.257171256013677,
    "p95_s_raw": 8.05358562800684
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 87.84887353749237,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 63.546620612477135
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 87.84887353749237
      },
      "calibration_gen": {
       "accuracy": 0.6435643564356436,
       "cc": 56.940570475966105,
       "cc_by_type": {
        "choice": 45.91728525980912,
        "noul": 76,
        "score": 48.9044261680892
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 56.940570475966105
      },
      "composite_gen": {
       "accuracy": 0.7821782178217822,
       "cc": 64.2194805783471,
       "cc_by_type": {
        "choice": 64.22764227642277,
        "noul": 55.55555555555556,
        "score": 72.875243903063
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 64.2194805783471
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 89.4731058334587,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 68.41931750037614
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 89.4731058334587
      },
      "jaggedness": {
       "accuracy": 0.6555555555555556,
       "cc": 51.15173057621822,
       "cc_by_type": {
        "choice": 53.801732435033685,
        "noul": 56.75675675675676,
        "score": 42.8967025368642
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 51.15173057621822
      },
      "llm_router": {
       "accuracy": 0.8370607028753994,
       "cc": 71.72047864231678,
       "cc_by_type": {
        "choice": 78.6035044465968,
        "noul": 87.62886597938144,
        "score": 48.92906550097207
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 71.72047864231678
      },
      "long_state": {
       "accuracy": 0.26666666666666666,
       "cc": -31.382028062735532,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": -100,
        "score": -3.8235035430453035
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5555555555555556,
       "cc": 31.665606480992682,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": 50,
        "score": 6.85248954607085
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 31.665606480992682
      },
      "multilingual": {
       "accuracy": 0.8,
       "cc": 69.17212800571714,
       "cc_by_type": {
        "choice": 75.54613628953375,
        "noul": 77.77777777777777,
        "score": 54.192469949839904
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.17212800571714
      },
      "negation_pair_gen": {
       "accuracy": 0.7790697674418605,
       "cc": 55.42410495913857,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 39.99999999999999,
        "score": 42.9389815440824
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 55.42410495913857
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 80.61760708445547,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100,
        "score": 62.873842274387414
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.61760708445547
      },
      "usecase": {
       "accuracy": 0.7272727272727273,
       "cc": 59.31205332037768,
       "cc_by_type": {
        "choice": 66.91176470588238,
        "noul": 58.33333333333333,
        "score": 52.691061921917324
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 59.31205332037768
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 24.545761058228774,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": -19.21985968245654
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 24.545761058228774
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 79.27519011200793,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 58.56289752496387
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 79.27519011200793
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 85.48969822591152,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 72.39829821755762
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 85.48969822591152
      },
      "el": {
       "accuracy": 0.8,
       "cc": 76.94633377862073,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 97.6918877418212
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 76.94633377862073
      },
      "en": {
       "accuracy": 0.7697654213727194,
       "cc": 64.4827040827019,
       "cc_by_type": {
        "choice": 71.00352692791152,
        "noul": 68.34532374100719,
        "score": 54.099261579186994
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 64.4827040827019
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.5487504069601,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 30.434179442740017
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.5487504069601
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 65.70550033996419,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 100,
        "score": 95.47715675759747
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.70550033996419
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 83.74288210729817,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 72.40511691012979
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 83.74288210729817
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 59.09382926533001,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 100,
        "score": -5.958735667697135
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 59.09382926533001
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 48.56239452957701,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -40.28319832865008
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 48.56239452957701
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 45.094346455841844,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 39.99999999999999,
        "score": 52.471284006127604
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 45.094346455841844
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 34.600352011262636,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 48.24550047823236
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 34.600352011262636
      },
      "mixed": {
       "accuracy": 1,
       "cc": 98.74153284520177,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.22459853560528
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 98.74153284520177
      },
      "nl": {
       "accuracy": 0.8,
       "cc": 48.51424476202488,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 12.209400952741323
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 48.51424476202488
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 71.6454228057721,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80,
        "score": 50.17113934410426
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 71.6454228057721
      },
      "pt": {
       "accuracy": 0.8571428571428571,
       "cc": 81.35337503204035,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100,
        "score": 65.27403199064081
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 81.35337503204035
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 37.10841544635961,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": 60.00000000000001,
        "score": 78.98482080716391
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 37.10841544635961
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 37.768161021349435,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -86.6955169359517
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 37.768161021349435
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 65.4423185501839,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -3.6730443494483067
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 65.4423185501839
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 37.32160001214934,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": -21.368533296885307
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 37.32160001214934
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.78,
       "cc": 71.5838420176429,
       "cc_by_type": {
        "choice": 71.5838420176429
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 71.5838420176429
      },
      "noul": {
       "accuracy": 0.856,
       "cc": 71.2,
       "cc_by_type": {
        "noul": 71.2
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 71.2
      },
      "score": {
       "accuracy": 0.672,
       "cc": 52.02832383682014,
       "cc_by_type": {
        "score": 52.02832383682014
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 52.02832383682014
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 39.183979710642106,
       "cc_by_type": {
        "choice": 68,
        "noul": 25,
        "score": 24.551939131926314
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 39.183979710642106
      },
      "code_linting": {
       "accuracy": 0.7083333333333334,
       "cc": 51.97732888297762,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 77.77777777777777,
        "score": -3.2122507561741065
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 51.97732888297762
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 42.34789039430175,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50,
        "score": -6.696166215468713
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 42.34789039430175
      },
      "demand_forecasting": {
       "accuracy": 0.5882352941176471,
       "cc": 40.03575445852952,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 33.33333333333333,
        "score": 45.253462205997906
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 40.03575445852952
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 30.104630020806553,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 67.12548426531822
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 30.104630020806553
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -40.6229056009301,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": -204.8875847273186
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 54.499913068666864,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 40.25043948611262
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 54.499913068666864
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 44.23391278756434,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 0,
        "score": 43.154700035166904
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 44.23391278756434
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 65.41586316119422,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 100,
        "score": 79.580922816916
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 65.41586316119422
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 44.35835074057644,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 30.648214248852803
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 44.35835074057644
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 72.23152413124878,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 100,
        "score": 28.322479370490505
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 72.23152413124878
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": -1.6851463390176065,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": -73.87264331812808
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 72.19021405060475,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 68.08695693108486
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 72.19021405060475
      },
      "model_routing": {
       "accuracy": 0.8755760368663594,
       "cc": 74.29999007860982,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 96.96969696969697,
        "score": 35.28092261678184
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.29999007860982
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 69.91229706647958,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 100,
        "score": 44.28583553725066
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 69.91229706647958
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 76.15530393930493,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 100,
        "score": 78.6051875839315
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 76.15530393930493
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 68.0715834514785,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 100,
        "score": 48.54972572389362
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 68.0715834514785
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 30.843083819184248,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": 33.33333333333333,
        "score": 60.645193486538254
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 30.843083819184248
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.96186264566033,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 95.56174452061086
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.96186264566033
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.1317029634105,
      88.88837341971802
     ],
     "I": [
      61.61085998902087,
      70.65774547957805
     ],
     "capability": [
      72.50621643606394,
      79.04031154182296
     ],
     "composite": null,
     "seed": 47
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.29383335481249,
     "capability": 76.89146782592724,
     "intelligence": 66.48910229704197
    },
    "per_type": {
     "choice": {
      "accuracy": 0.78,
      "calibration": 89.87280726007195,
      "cc": 71.5838420176429,
      "cc_by_type": {
       "choice": 71.5838420176429
      },
      "cc_tiered": 74.078893695015,
      "cc_tiered_open": 79.70868696671687,
      "cc_tiered_sealed": 68.44910042331313,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 71.5838420176429
     },
     "noul": {
      "accuracy": 0.856,
      "calibration": 88.19315315315316,
      "cc": 71.2,
      "cc_by_type": {
       "noul": 71.2
      },
      "cc_tiered": 73.07647309664586,
      "cc_tiered_open": 79.59903381642512,
      "cc_tiered_sealed": 66.55391237686658,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 71.2
     },
     "score": {
      "accuracy": 0.672,
      "calibration": 83.81553965121238,
      "cc": 52.02832383682014,
      "cc_by_type": {
       "score": 52.02832383682014
      },
      "cc_tiered": 52.31194009946505,
      "cc_tiered_open": 54.28293661429005,
      "cc_tiered_sealed": 50.340943584640044,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.02832383682014
     }
    },
    "raw_on_A": {
     "calibration": 85.53121656093457,
     "capability": 75.98626480220324,
     "intelligence": 66.44131304347191
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.010666666666666713,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 67.7039298571889,
    "Emphasis on Accuracy 60:20:20": 67.21271039320483,
    "Emphasis on Cost 20:20:60": 66.67327266812579,
    "Emphasis on Speed 20:60:20": 69.2812390030769,
    "Intelligence only": 66.48910229704197
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "MeTask (Hangzhou MeTask Technology Co., Ltd.)",
   "axes": {
    "calibration": 87.09471194592824,
    "cost": 56.13017070777345,
    "intelligence": 70.71905450578407,
    "speed": 82.33528699876703
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.057130297017528,
      "ece_score": 88.5739405964944,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 88.5739405964944,
      "score_before_time_limit": 88.5739405964944,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.1123272461650836,
      "ece": 0.06861261261261234,
      "n": 375,
      "score": 86.27747747747753,
      "score_before_time_limit": 86.27747747747753,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9226666666666666,
      "interval90_width": 0.2508888888888888,
      "n": 375,
      "nrps": 0.07405460121549541,
      "score": 86.43271776381279,
      "score_before_time_limit": 86.43271776381279,
      "time_limit_failed_share": 0,
      "top_ece": 0.09864552175412443
     }
    },
    "score": 87.09471194592824
   },
   "capability": 78.90688322585615,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 718,318 input + 1,485 output tokens over 1,485 decisions = 484.715824916 tokens/decision. GPU-time cost USD 0.0289953024393587/1,000; old list-price cost USD 0.0438744242424242/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0289953024393587/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.04921078
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.028995302439358734
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "metask-jev-rain-31B-trained",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline read-only container (--network none); author's own service (service/serve.sh -> launch.py -> server.py on vLLM 0.31.0, pinned digest image), unchanged defaults paid evaluation (public). Own LoRA adapter (rank 16) on Google's official W4A16 Gemma 4 31B QAT checkpoint, applied unmerged by vLLM; max_model_len 6656, longer inputs are shortened by the service's documented extraction rule (author-tested only on H200; measured on H100 80GB with unchanged settings).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.5520469583762,
    "I_sealed": 68.88606205319194,
    "base": 70.71905450578407,
    "excess": -6.173541865171451,
    "gap": 3.66598490518426,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 80.81042171537538,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 76.26895187870798,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 73.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 63.24668534332811,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 70.41765078820325,
       "hard": 69.57893658072337,
       "judge": 60.72068725487829,
       "standard": 50.785697278774734
      }
     },
     "sealed|choice": {
      "cc": 80.73173886349481,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 78.142708453799,
       "judge": 78.84427032321254,
       "standard": 81.96582999367222
      }
     },
     "sealed|noul": {
      "cc": 74.61818469744357,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 67.27272727272727,
       "judge": 81.53846153846153,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 51.30826259863744,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 70.15351391347656,
       "hard": 51.34670276898963,
       "judge": 46.83933293506929,
       "standard": 48.51215109586572
      }
     }
    }
   },
   "jevbench_score": 71.96392483986129,
   "key": "metask_jev_rain_31b_trained",
   "last_measured_on": "2026-10-12",
   "licence": "Weights: Apache-2.0 per model card (base Gemma 4 under the Gemma 4 licence); service code (service/) MIT",
   "listing": "ranked",
   "measured_in": "v1.6.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "MeTask-Rain/metask-jev-rain-31B-trained@da56acc0bb607cc214c184ad9f1e9231a0d63dee",
   "noul_decisive": {
    "acc_among_decisive": 0.8773333333333333,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.09471194592824,
     "cost": 50.73365709012802,
     "intelligence": 70.71905450578407,
     "speed": 83.17884674032656
    },
    "board_release": "v1.6.43",
    "capability": 78.90688322585615,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 8,
    "capability_rank": 15,
    "composite_ci95": {
     "A": [
      67.97744875057778,
      70.7750334864619
     ],
     "B": [
      67.57519316040789,
      71.5173477277133
     ],
     "C": [
      67.97744875057778,
      70.7750334864619
     ]
    },
    "jevbench_score": 69.74074308829535,
    "measured_in": "v1.6.10",
    "method": "v1.6",
    "open_board_rank": 12,
    "rank": 28,
    "ranks": {
     "A": 28,
     "B": 21,
     "C": 26,
     "capability": 30
    },
    "scores": {
     "A": 69.74074308829535,
     "B": 69.93423396579702,
     "C": 69.74074308829535
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5687712740385905,
     "p50_s_raw": 0.20938563701929525,
     "p95_s_adjusted": 0.8456247233552858,
     "p95_s_raw": 0.3478123616776429
    },
    "views": {
     "Balanced 33:33:33": 65.39719257145047,
     "Emphasis on Accuracy 60:20:20": 67.42683688195376,
     "Emphasis on Cost 20:20:60": 58.62002091885354,
     "Emphasis on Speed 20:60:20": 71.51223704651308,
     "Intelligence only": 70.71905450578407
    }
   },
   "rank": 58,
   "ranked": true,
   "ranks": {
    "A": 58,
    "B": 40,
    "C": 45,
    "capability": 40
   },
   "repo": "https://huggingface.co/MeTask-Rain/metask-jev-rain-31B-trained",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.96392483986129,
    "B": 71.7114569132223,
    "C": 71.96392483986129
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.6160417439881712,
    "p50_s_raw": 0.2330208719940856,
    "p95_s_adjusted": 0.9481153076398185,
    "p95_s_raw": 0.39905765381990926
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 63.16886779201999,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 14.506603376059957
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 63.16886779201999
      },
      "calibration_gen": {
       "accuracy": 0.7722772277227723,
       "cc": 64.53799979415957,
       "cc_by_type": {
        "choice": 77.7306468716861,
        "noul": 60.00000000000001,
        "score": 55.88335251079264
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 64.53799979415957
      },
      "composite_gen": {
       "accuracy": 0.8811881188118812,
       "cc": 83.81438823065723,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 62.102604619704316
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 83.81438823065723
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.04066368360208,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 98.12199105080622
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.04066368360208
      },
      "jaggedness": {
       "accuracy": 0.6666666666666666,
       "cc": 49.85525524192105,
       "cc_by_type": {
        "choice": 62.463907603464875,
        "noul": 51.35135135135136,
        "score": 35.750506770946934
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 49.85525524192105
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 83.7793054809374,
       "cc_by_type": {
        "choice": 93.66029761380645,
        "noul": 95.8762886597938,
        "score": 61.801330169211944
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.7793054809374
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": 9.832941925502283,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": 0,
        "score": 19.821406421668144
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 9.832941925502283
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 54.59598157228413,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 75,
        "score": 25.901346778708078
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 54.59598157228413
      },
      "multilingual": {
       "accuracy": 0.7963636363636364,
       "cc": 68.20002344704231,
       "cc_by_type": {
        "choice": 75.54613628953375,
        "noul": 69.44444444444444,
        "score": 59.60948960714872
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 68.20002344704231
      },
      "negation_pair_gen": {
       "accuracy": 0.872093023255814,
       "cc": 68.5323347023584,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 60.00000000000001,
        "score": 52.263670773741836
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 68.5323347023584
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 81.60522267243225,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 59.83068303231177
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 81.60522267243225
      },
      "usecase": {
       "accuracy": 0.7363636363636363,
       "cc": 64.54982362341306,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 70.83333333333333,
        "score": 46.100451262396035
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 64.54982362341306
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 36.07646009918423,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": -3.675381607209216
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 36.07646009918423
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 73.84108691654622,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 42.260587938578766
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 73.84108691654622
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 90.42477813573423,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 87.20353794702571
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 90.42477813573423
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.579631749179,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.59178165349604
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.579631749179
      },
      "en": {
       "accuracy": 0.8227628149435273,
       "cc": 71.3569930395452,
       "cc_by_type": {
        "choice": 83.0853907079484,
        "noul": 75.53956834532374,
        "score": 55.44602006536346
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 71.3569930395452
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 69.16556628426598,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 27.07255529651733
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 69.16556628426598
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 88.9244647500369,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": 99.56027949601237
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 88.9244647500369
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 93.75128961226879,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 81.25386883680636
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 93.75128961226879
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 73.31989019858385,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100,
        "score": 19.959670595751543
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 73.31989019858385
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 56.92598694485033,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -15.192421082830165
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 56.92598694485033
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 46.34982921310412,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 80,
        "score": 1.9406711182639036
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 46.34982921310412
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 60.502271657899314,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 81.50681497369794
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 60.502271657899314
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.73180568607792,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.86208372490044
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.73180568607792
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 8.642558761906725,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 11.209061567105461
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 8.642558761906725
      },
      "no": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 100
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 63.392816051625765,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 57.79588361827128
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 63.392816051625765
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 67.80504865797501,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": 56.44991321022436
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 67.80504865797501
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 46.19054683365891,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 29.63547028821076
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 46.19054683365891
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 68.75012962289753,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 27.989519303475184
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 68.75012962289753
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 61.68878465176133,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 49.582482987542065
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 61.68878465176133
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 69.21114832796756,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 7.63344498390266
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 69.21114832796756
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8573333333333333,
       "cc": 81.57255209628963,
       "cc_by_type": {
        "choice": 81.57255209628963
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.57255209628963
      },
      "noul": {
       "accuracy": 0.8773333333333333,
       "cc": 75.46666666666665,
       "cc_by_type": {
        "noul": 75.46666666666665
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 75.46666666666665
      },
      "score": {
       "accuracy": 0.6746666666666666,
       "cc": 54.7684541832536,
       "cc_by_type": {
        "score": 54.7684541832536
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 54.7684541832536
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 48.45861881975295,
       "cc_by_type": {
        "choice": 84,
        "noul": 25,
        "score": 36.375856459258834
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 48.45861881975295
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 76.75345303484892,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 30.26035910454675
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 76.75345303484892
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 69.6944643957582,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100,
        "score": 25.343555788900616
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 69.6944643957582
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -24.108890369968993,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -33.333333333333336,
        "score": -33.7301798818368
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 57.255300551831404,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 67.41807556853769
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 57.255300551831404
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 35.41674609079076,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 66.66666666666667,
        "score": -35.25919568989187
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 35.41674609079076
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 56.61757692065063,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 71.42857142857142,
        "score": 27.835924039262792
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 56.61757692065063
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 43.298609533332666,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0,
        "score": 61.2547136174196
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 43.298609533332666
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 40.88581737462815,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 25.15745212388444
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 40.88581737462815
      },
      "knowledge_graphs": {
       "accuracy": 0.7727272727272727,
       "cc": 70.06051887452556,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100,
        "score": 35.877488101092766
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 70.06051887452556
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 64.06806981529785,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 14.426431668115768
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 64.06806981529785
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 49.78201560348273,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100,
        "score": 46.1202403588353
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 49.78201560348273
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 72.7024042480826,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 58.107212744247796
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 72.7024042480826
      },
      "model_routing": {
       "accuracy": 0.9032258064516129,
       "cc": 80.21287919965458,
       "cc_by_type": {
        "choice": 91.81818181818183,
        "noul": 100,
        "score": 48.82045578078191
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.21287919965458
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 48.98671618683705,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 69.9927781190524
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 48.98671618683705
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 78.10700850764788,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 100,
        "score": 67.74720936695476
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 78.10700850764788
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 56.67924976273154,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 100,
        "score": -0.40560046549995565
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 56.67924976273154
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 45.24701491441865,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 45.88597227948784
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 45.24701491441865
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.61559815097014,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.84679445291042
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.61559815097014
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.90626522739525,
      88.6737670940263
     ],
     "I": [
      65.93306472259675,
      74.78359254040711
     ],
     "capability": [
      74.80944444127567,
      81.28198974405463
     ],
     "composite": null,
     "seed": 48
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.09471194592824,
     "capability": 78.90688322585615,
     "intelligence": 70.71905450578407
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8573333333333333,
      "calibration": 88.5739405964944,
      "cc": 81.57255209628963,
      "cc_by_type": {
       "choice": 81.57255209628963
      },
      "cc_tiered": 80.7710802894351,
      "cc_tiered_open": 80.81042171537538,
      "cc_tiered_sealed": 80.73173886349481,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.57255209628963
     },
     "noul": {
      "accuracy": 0.8773333333333333,
      "calibration": 86.27747747747753,
      "cc": 75.46666666666665,
      "cc_by_type": {
       "noul": 75.46666666666665
      },
      "cc_tiered": 74.10860925693434,
      "cc_tiered_open": 73.59903381642512,
      "cc_tiered_sealed": 74.61818469744357,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 75.46666666666665
     },
     "score": {
      "accuracy": 0.6746666666666666,
      "calibration": 86.43271776381279,
      "cc": 54.7684541832536,
      "cc_by_type": {
       "score": 54.7684541832536
      },
      "cc_tiered": 57.277473970982776,
      "cc_tiered_open": 63.24668534332811,
      "cc_tiered_sealed": 51.30826259863744,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.7684541832536
     }
    },
    "raw_on_A": {
     "calibration": 83.28617618857919,
     "capability": 76.3197068643228,
     "intelligence": 69.35323754006639
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.02466275345748,
    "Emphasis on Accuracy 60:20:20": 69.07740068945235,
    "Emphasis on Cost 20:20:60": 62.709203276606274,
    "Emphasis on Speed 20:60:20": 73.10735278026625,
    "Intelligence only": 70.71905450578407
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "MeTask (Hangzhou MeTask Technology Co., Ltd.)",
   "axes": {
    "calibration": 86.60202339920977,
    "cost": 55.40100072278519,
    "intelligence": 72.59243765084337,
    "speed": 73.27177592289428
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06613866866964949,
      "ece_score": 86.7722662660701,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 86.7722662660701,
      "score_before_time_limit": 86.7722662660701,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.1016786666666672,
      "ece": 0.06551999999999943,
      "n": 375,
      "score": 86.89600000000011,
      "score_before_time_limit": 86.89600000000011,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.92,
      "interval90_width": 0.20799999999999996,
      "n": 375,
      "nrps": 0.06738985445027339,
      "score": 86.13780393155909,
      "score_before_time_limit": 86.13780393155909,
      "time_limit_failed_share": 0,
      "top_ece": 0.10492703345927233
     }
    },
    "score": 86.60202339920977
   },
   "capability": 79.59723052502656,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 743,854 input + 17,382 output tokens over 1,485 decisions = 512.616835017 tokens/decision. GPU-time cost USD 0.0306643179421698/1,000; old list-price cost USD 0.0490617777777778/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0306643179421698/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05434626
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03066431794216985
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "metask-jev-rain-31B-trained service v3",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX PRO 6000 96 GB; the reviewed recipe needs 96 GB), offline read-only container (--network none, verified by docker inspect before the input upload); the author's own service at revision 2feff076 (service v3), unchanged defaults. paid evaluation. RE-SUBMISSION, DISCLOSED: the same LoRA weights as the published metask_jev_rain_31b_trained row (paid evaluation, v1.6.10) with a changed service/ layer (e.g. its yes/no constant); those changes may have been tuned on that row's published v1.6.10 aggregates (under the pre-registration). Both rows are shown; no third same-weights row is admitted on this draw. Weights file shas: model 1b9b1d62..., adapter 2b956899..., code tree 6a1234ee...",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.52301075937709,
    "I_sealed": 72.66186454230964,
    "base": 72.59243765084337,
    "excess": -9.978380553288268,
    "gap": -0.13885378293255712,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 80.81042171537538,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 76.26895187870798,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 73.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 63.15957674633078,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.16584682233966,
       "hard": 69.17688425857045,
       "judge": 60.01699079087648,
       "standard": 50.83570561702846
      }
     },
     "sealed|choice": {
      "cc": 82.58817768397486,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 80.2243552677229,
       "judge": 79.72575905974536,
       "standard": 85.76249736342545
      }
     },
     "sealed|noul": {
      "cc": 76.53948727546258,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 69.0909090909091,
       "judge": 81.53846153846153,
       "standard": 79.10447761194031
      }
     },
     "sealed|score": {
      "cc": 58.85792866749151,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 69.80503007296836,
       "hard": 56.95507418481911,
       "judge": 61.45515272353761,
       "standard": 53.2942508460287
      }
     }
    }
   },
   "jevbench_score": 70.15002550079586,
   "key": "metask_jev_rain_31b_trained_v3",
   "last_measured_on": "2026-10-13",
   "licence": "Weights: Apache-2.0 per model card (base Gemma 4 under the Gemma 4 licence); service code (service/) MIT",
   "listing": "ranked",
   "measured_in": "v1.6.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "MeTask-Rain/metask-jev-rain-31B-trained@2feff076f07711fe2fefe7a6fdcedfe4132b773e",
   "noul_decisive": {
    "acc_among_decisive": 0.8853333333333333,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.60202339920977,
     "cost": 49.27770156907625,
     "intelligence": 72.59243765084337,
     "speed": 84.66561996644248
    },
    "board_release": "v1.6.43",
    "capability": 79.59723052502656,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 6,
    "capability_rank": 13,
    "composite_ci95": {
     "A": [
      66.27480907834259,
      68.67789744912794
     ],
     "B": [
      66.21154989482208,
      69.66970736124362
     ],
     "C": [
      66.27480907834259,
      68.67789744912794
     ]
    },
    "jevbench_score": 67.6558274201148,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": 21,
    "rank": 41,
    "ranks": {
     "A": 41,
     "B": 27,
     "C": 34,
     "capability": 25
    },
    "scores": {
     "A": 67.6558274201148,
     "B": 68.20807250966563,
     "C": 67.6558274201148
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5115797939710319,
     "p50_s_raw": 0.18078989698551595,
     "p95_s_adjusted": 0.6676127476617694,
     "p95_s_raw": 0.2588063738308847
    },
    "views": {
     "Balanced 33:33:33": 63.51261802145979,
     "Emphasis on Accuracy 60:20:20": 66.13810967705214,
     "Emphasis on Cost 20:20:60": 56.167442462815025,
     "Emphasis on Speed 20:60:20": 69.87668805746007,
     "Intelligence only": 70.5102505244786
    }
   },
   "rank": 76,
   "ranked": true,
   "ranks": {
    "A": 76,
    "B": 48,
    "C": 59,
    "capability": 36
   },
   "repo": "https://huggingface.co/MeTask-Rain/metask-jev-rain-31B-trained",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.15002550079586,
    "B": 70.62527064951077,
    "C": 70.15002550079586
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5195755619555712,
    "p50_s_raw": 0.1847877809777856,
    "p95_s_adjusted": 9.06094911987893,
    "p95_s_raw": 4.455474559939465
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 89.03475546368588,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 83.77093305772432
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 89.03475546368588
      },
      "calibration_gen": {
       "accuracy": 0.7920792079207921,
       "cc": 68.68645561782115,
       "cc_by_type": {
        "choice": 77.7306468716861,
        "noul": 68,
        "score": 60.328719981777354
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 68.68645561782115
      },
      "composite_gen": {
       "accuracy": 0.8910891089108911,
       "cc": 84.77793164693051,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 64.99323486852417
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 84.77793164693051
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.04372227473641,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 98.13116682420925
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.04372227473641
      },
      "jaggedness": {
       "accuracy": 0.7,
       "cc": 55.63506291104909,
       "cc_by_type": {
        "choice": 69.68238691049086,
        "noul": 56.75675675675676,
        "score": 40.46604506589963
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 55.63506291104909
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 83.60538459908456,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 95.8762886597938,
        "score": 60.48710472537918
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.60538459908456
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": 8.720258887926093,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": 0,
        "score": 16.483357308939574
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 8.720258887926093
      },
      "ml_usecase": {
       "accuracy": 0.8148148148148148,
       "cc": 62.009380987023235,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 75,
        "score": 48.14154502292535
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 62.009380987023235
      },
      "multilingual": {
       "accuracy": 0.8145454545454546,
       "cc": 70.77751409077383,
       "cc_by_type": {
        "choice": 77.50244538637104,
        "noul": 72.22222222222223,
        "score": 62.60787466372824
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 70.77751409077383
      },
      "negation_pair_gen": {
       "accuracy": 0.8837209302325582,
       "cc": 71.91781286399173,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 60.00000000000001,
        "score": 62.420105258641826
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 71.91781286399173
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 82.80777431471158,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 63.43833795914975
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 82.80777431471158
      },
      "usecase": {
       "accuracy": 0.7772727272727272,
       "cc": 68.49729252167786,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 70.83333333333333,
        "score": 57.942857957190455
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 68.49729252167786
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 43.24855438365077,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": 17.84090124619041
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 43.24855438365077
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 74.01835450650829,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 42.79239070846494
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 74.01835450650829
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 81.83435068683548,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 61.432255600329455
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 81.83435068683548
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.60222006784046,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.65954660948044
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.60222006784046
      },
      "en": {
       "accuracy": 0.8410078192875761,
       "cc": 74.40408858051423,
       "cc_by_type": {
        "choice": 84.40341221122515,
        "noul": 76.97841726618707,
        "score": 61.83043626413047
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 74.40408858051423
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 68.2680286109703,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 24.379942276630317
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 68.2680286109703
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 88.93547668805367,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": 99.59331531006269
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 88.93547668805367
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 93.7369976649519,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 81.2109929948557
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 93.7369976649519
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 80.07784582404595,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100,
        "score": 40.23353747213787
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 80.07784582404595
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 54.53760923768931,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -22.357554204313203
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 54.53760923768931
      },
      "ja": {
       "accuracy": 0.84,
       "cc": 64.88034315630443,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 80,
        "score": 28.938090628563796
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 64.88034315630443
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 60.26417092788616,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 80.79251278365848
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 60.26417092788616
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.93609912586727,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.80829737760182
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.93609912586727
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 8.53215978329243,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 10.877864631262579
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 8.53215978329243
      },
      "no": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 100
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 63.14426530245589,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 57.050231370761686
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 63.14426530245589
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 74.38219774143825,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 65.57440701335422
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 74.38219774143825
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 46.19054710826382,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 29.635471112025513
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 46.19054710826382
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 69.0713909756119,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 28.95330336161831
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 69.0713909756119
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 61.68232554763859,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 49.56310567517384
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 61.68232554763859
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 69.21240428354913,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 7.637212850647435
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 69.21240428354913
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8693333333333333,
       "cc": 83.12252434987275,
       "cc_by_type": {
        "choice": 83.12252434987275
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 83.12252434987275
      },
      "noul": {
       "accuracy": 0.8853333333333333,
       "cc": 77.06666666666666,
       "cc_by_type": {
        "noul": 77.06666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 77.06666666666666
      },
      "score": {
       "accuracy": 0.7173333333333334,
       "cc": 60.018133778302385,
       "cc_by_type": {
        "score": 60.018133778302385
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 60.018133778302385
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 49.15314922895124,
       "cc_by_type": {
        "choice": 84,
        "noul": 25,
        "score": 38.45944768685374
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 49.15314922895124
      },
      "code_linting": {
       "accuracy": 0.8333333333333334,
       "cc": 80.079808393905,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 40.239425181714985
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 80.079808393905
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 84.23287684117527,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 85.21895572677785
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 84.23287684117527
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -20.319424506293327,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -33.333333333333336,
        "score": -22.361782290809806
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 57.24850688656365,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 67.39769457273441
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 57.24850688656365
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 30.523095261874122,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 66.66666666666667,
        "score": -49.940148176641785
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 30.523095261874122
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 60.33576603025934,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 71.42857142857142,
        "score": 38.99049136808895
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 60.33576603025934
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 46.327230880637806,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 0,
        "score": 70.340577659335
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 46.327230880637806
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 54.927785267620116,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 67.28335580286033
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 54.927785267620116
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 78.22201831947774,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100,
        "score": 47.51402069719127
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 78.22201831947774
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 63.998217861086424,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 14.216875805481488
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 63.998217861086424
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 51.14449916437249,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100,
        "score": 50.207691041504575
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 51.14449916437249
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 75.39671275196183,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 66.19013825588543
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 75.39671275196183
      },
      "model_routing": {
       "accuracy": 0.9078341013824884,
       "cc": 80.45531443691877,
       "cc_by_type": {
        "choice": 92.98701298701299,
        "noul": 100,
        "score": 48.37893032374332
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.45531443691877
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 56.46101690387395,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 92.4156802701631
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 56.46101690387395
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 82.04193370234395,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 100,
        "score": 79.55198495104301
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 82.04193370234395
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 56.88699086990223,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 100,
        "score": 0.21762285601210074
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 56.88699086990223
      },
      "scientific_discovery": {
       "accuracy": 0.6956521739130435,
       "cc": 47.859636984518865,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 53.7238384897885
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 47.859636984518865
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.6261843392683,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.87855301780489
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.6261843392683
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.1062803062735,
      89.00397681987438
     ],
     "I": [
      67.96927371270253,
      76.63101848971161
     ],
     "capability": [
      76.2469833472818,
      82.24205839797754
     ],
     "composite": null,
     "seed": 49
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.60202339920977,
     "capability": 79.59723052502656,
     "intelligence": 72.59243765084337
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8693333333333333,
      "calibration": 86.7722662660701,
      "cc": 83.12252434987275,
      "cc_by_type": {
       "choice": 83.12252434987275
      },
      "cc_tiered": 81.69929969967512,
      "cc_tiered_open": 80.81042171537538,
      "cc_tiered_sealed": 82.58817768397486,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.12252434987275
     },
     "noul": {
      "accuracy": 0.8853333333333333,
      "calibration": 86.89600000000011,
      "cc": 77.06666666666666,
      "cc_by_type": {
       "noul": 77.06666666666666
      },
      "cc_tiered": 75.06926054594385,
      "cc_tiered_open": 73.59903381642512,
      "cc_tiered_sealed": 76.53948727546258,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 77.06666666666666
     },
     "score": {
      "accuracy": 0.7173333333333334,
      "calibration": 86.13780393155909,
      "cc": 60.018133778302385,
      "cc_by_type": {
       "score": 60.018133778302385
      },
      "cc_tiered": 61.00875270691114,
      "cc_tiered_open": 63.15957674633078,
      "cc_tiered_sealed": 58.85792866749151,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 60.018133778302385
     }
    },
    "raw_on_A": {
     "calibration": 81.92265838998793,
     "capability": 76.06423959692178,
     "intelligence": 70.20582080385563
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.97238073528067,
    "Emphasis on Accuracy 60:20:20": 68.47002642174861,
    "Emphasis on Cost 20:20:60": 61.29403534996144,
    "Emphasis on Speed 20:60:20": 68.71037457361517,
    "Intelligence only": 72.59243765084337
   }
  },
  {
   "adapter_id": "images/mghafiri-qwen3.5-0.8b-decision-model/spec.json:mghafiri_adapter.JevLite",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mourad Ghafiri",
   "axes": {
    "calibration": 77.95657740696296,
    "cost": 86.07480552202115,
    "intelligence": 7.297270616183145,
    "speed": 55.90747410573378
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.16652875000000003,
      "ece_score": 66.69425,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 66.69425
     },
     "noul": {
      "abstention_rate": 0.7813333333333333,
      "brier": 0.20344673173333316,
      "ece": 0.05893599999999998,
      "n": 375,
      "score": 88.2128
     },
     "score": {
      "interval90_coverage": 0.8586666666666667,
      "interval90_width": 0.6119999999999997,
      "n": 375,
      "nrps": 0.16765142224888893,
      "score": 78.9626822208889,
      "top_ece": 0.12654746666666666
     }
    },
    "score": 77.95657740696296
   },
   "capability": 42.62692401157305,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.8B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 782,199 input + 0 output tokens over 1,624 decisions = 481.649630542 tokens/decision. GPU-time cost USD 0.00291187782591485/1,000; old list-price cost USD 0.00481649630541872/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00291187782591485/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0029118778259148497
   },
   "decision_limit": null,
   "display": "Qwen3.5-0.8B Decision Model (Mourad Ghafiri)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 4.79058394567308,
    "I_sealed": 9.803957286693208,
    "base": 7.297270616183145,
    "excess": -7.5867609834583725,
    "gap": -5.013373341020128,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 13.339251237806783,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -1.1891891891891837,
       "hard": 13.180245360176167,
       "judge": 20.896506262359924,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -73.91304347826086,
       "judge": -90,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 1.0325005992124563,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 40.642727272727264,
       "hard": -20.778611111111122,
       "judge": 25.48779661016949,
       "standard": -11.83333333333334
      }
     },
     "sealed|choice": {
      "cc": 21.186916712409108,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 24.174118690414627,
       "hard": 20.079185520361975,
       "judge": 20.856610800744896,
       "standard": 22.404236974996934
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -60,
       "hard": -68.93203883495146,
       "judge": -57.89473684210527,
       "standard": -43.66197183098591
      }
     },
     "sealed|score": {
      "cc": 8.224955147670517,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 42.56900709219856,
       "hard": 1.4315441176470767,
       "judge": 8.896227544910184,
       "standard": 3.6328426395938807
      }
     }
    }
   },
   "jevbench_score": 0.47499805397335026,
   "key": "mghafiri-qwen3.5-0.8b-decision-model",
   "last_measured_on": "2026-10-02",
   "licence": "MIT code and training data; Apache-2.0 model weights",
   "listing": "remeasure-pending",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.6",
   "model_pin": null,
   "not_ranked_because": "Re-measure pending under the 10 s rule: this row's per-decision times were recorded on a loaded CPU host, so they cannot be scored under the time limit. Shown with its v1.6 values until it is measured again.",
   "noul_decisive": {
    "acc_among_decisive": 0.8780487804878049,
    "decisive_rate": 0.21866666666666668,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.95657740696296,
     "cost": 79.5180630553356,
     "intelligence": 7.297270616183145,
     "speed": 55.90747410573378
    },
    "board_release": "v1.6.43",
    "capability": 42.62692401157305,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.18025356076996352,
      1.2178442350406786
     ],
     "B": [
      0.12404410707015784,
      0.9011760636632091
     ],
     "C": [
      0.1251760838680302,
      0.8457251632226935
     ]
    },
    "jevbench_score": 0.47247471382875306,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 164,
    "rank": 207,
    "ranks": {
     "A": 207,
     "B": 207,
     "C": 207,
     "capability": 200
    },
    "scores": {
     "A": 0.47247471382875306,
     "B": 0.3355765591935497,
     "C": 0.3281074401588563
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 11.252091223932803,
     "p50_s_raw": 5.551045611966401,
     "p95_s_adjusted": 22.804436485376208,
     "p95_s_raw": 11.327218242688105
    },
    "views": {
     "Balanced 33:33:33": 0.381493687842595,
     "Emphasis on Accuracy 60:20:20": 0.24118281186608428,
     "Emphasis on Cost 20:20:60": 0.5528132957063066,
     "Emphasis on Speed 20:60:20": 0.5239265172020665,
     "Intelligence only": 0.15543232661348225
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/mghafiri/qwen3.5-0.8B-decision-model",
   "scores": {
    "A": 0.47499805397335026,
    "B": 0.3365925540381815,
    "C": 0.32985975970371545
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "n": 124,
    "p50_s_adjusted": 11.252091223932803,
    "p50_s_raw": 5.551045611966401,
    "p95_s_adjusted": 22.804436485376208,
    "p95_s_raw": 11.327218242688105
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-0.8B-Base fine-tuned as a JevLite decision model with per-question calibration",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.68948470123519,
      80.02198153079908
     ],
     "I": [
      5.167522017117285,
      10.33633269787795
     ],
     "capability": [
      38.95895177314971,
      44.525366548573615
     ],
     "composite": null,
     "seed": 67
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.95657740696296,
     "capability": 42.62692401157305,
     "intelligence": 7.297270616183145
    },
    "per_type": {
     "choice": {
      "accuracy": 0.36933333333333335,
      "calibration": 66.69425,
      "cc": 19.346740808806086,
      "cc_by_type": {
       "choice": 19.346740808806086
      },
      "cc_tiered": 17.263083975107946,
      "cc_tiered_open": 13.339251237806783,
      "cc_tiered_sealed": 21.186916712409108,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 19.346740808806086
     },
     "noul": {
      "accuracy": 0.192,
      "calibration": 88.2128,
      "cc": -61.6,
      "cc_by_type": {
       "noul": -61.6
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4106666666666667,
      "calibration": 78.9626822208889,
      "cc": 7.9552982456141175,
      "cc_by_type": {
       "score": 7.9552982456141175
      },
      "cc_tiered": 4.628727873441487,
      "cc_tiered_open": 1.0325005992124563,
      "cc_tiered_sealed": 8.224955147670517,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 7.9552982456141175
     }
    },
    "raw_on_A": {
     "calibration": 67.24894909368233,
     "capability": 38.149863284895126,
     "intelligence": 9.050777476107926
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.3836880574062214,
    "Emphasis on Accuracy 60:20:20": 0.24170717394641789,
    "Emphasis on Cost 20:20:60": 0.5611847513261395,
    "Emphasis on Speed 20:60:20": 0.5264072938259837,
    "Intelligence only": 0.15543232661348225
   }
  },
  {
   "api_flag": true,
   "author": "Microsoft",
   "axes": {
    "calibration": 84.39873071167106,
    "cost": 61.452075716621465,
    "intelligence": 57.16958510591244,
    "speed": 81.38546065308665
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09636174358194516,
      "ece_score": 80.72765128361097,
      "mean_tvd": null,
      "n": 735,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 80.72765128361097,
      "score_before_time_limit": 80.72765128361097,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2278820375335121,
      "brier": 0.10202697897728127,
      "ece": 0.05824671413738715,
      "n": 373,
      "score": 88.35065717252257,
      "score_before_time_limit": 88.35065717252257,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9056603773584906,
      "interval90_width": 0.21495956873315356,
      "n": 371,
      "nrps": 0.07658334569212163,
      "score": 84.11788367887968,
      "score_before_time_limit": 84.11788367887968,
      "time_limit_failed_share": 0,
      "top_ece": 0.12052949036514232
     }
    },
    "score": 84.39873071167106
   },
   "capability": 70.78415790879176,
   "capability_ci95": [
    65.79924499904618,
    73.0808555865328
   ],
   "composite_ci95": {
    "A": [
     65.99814392729574,
     70.37976430138654
    ],
    "B": [
     62.310798268533894,
     68.07228955990982
    ],
    "C": [
     46.72503296526089,
     70.24484593257228
    ]
   },
   "cost": {
    "basis": "Advertised Azure Foundry tariff .042/M input, output free; genuine native usage, billed invoice unknown",
    "common_basis": "{\"n_items\": 1479, \"rule\": \"common\", \"usd_per_1000_all_items\": 0.019002368000000002}",
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0192721784989858
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Microsoft-Decision-1 (Azure Foundry)",
   "endpoint_kind": "api",
   "fresh_api_addendum": {
    "acceptance_sha256": "1a21b7d132e77af0196edebb1c4f9060e2eb3067aa04ec5ed99cb94bdbc808ce",
    "categories_sha256": "47a4db8923dc9edadcd74ee357c852099a7473e8f6b99100e8fe61767b9e63c5",
    "cost_mask_sha256": "50d582d3976ba79d458756f931764cf6e80bc623fbbe2fa611d8744752e3fc7b",
    "cost_n": 1479,
    "counts": {
     "P": 300,
     "S": 1200
    },
    "g_med_s": 7.088097744249485,
    "input_sha256": "6833e921ae525a328a4cf199a3e8145fa657fe2c9a9fe520e2953aab6da60ee6",
    "method": "O1S",
    "parent": "v1.6-fastlane-20261009",
    "release": "CR-401 Microsoft-Decision-1 full native API addendum",
    "release_authorization_sha256": "ea7e7d50d31ba80c47bdba27f9b32f35f03dabfd2bb1fe6264f6e9d5588023a3",
    "run_sha256": "cadf99e94de455277604759b31028cfb14663f3b815555e5ea7909e0bb891c27",
    "scorer_sha256": "83398383da9d011cc30236d0a241a9c125fc5efbbfc041f5564e7c5f699d896e",
    "source_url": "https://benchmarkheaven.com/api/jevbench/api-addenda"
   },
   "intelligence": {
    "I_open": 65.54484720922025,
    "I_sealed": 49.68435126148425,
    "base": 57.61459923535225,
    "excess": 8.772398203486514,
    "gap": 15.860495947735998,
    "penalty": 0.9922760179651349,
    "per_type_split": {
     "open|choice": {
      "cc": 82.42557001849185,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 75.46398238439761,
       "judge": 84.17930125247199,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 51.46376811594203,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 47.82608695652173,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 62.74520349322689,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.19290622964672,
       "hard": 67.62102226028924,
       "judge": 74.3930948447687,
       "standard": 41.29787756357959
      }
     },
     "sealed|choice": {
      "cc": 65.77933402862911,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 85.18366087045993,
       "hard": 59.4822734946539,
       "judge": 65.37585421412302,
       "standard": 69.27651139742319
      }
     },
     "sealed|noul": {
      "cc": 37.62824744344246,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 54.16666666666667,
       "hard": 25.86206896551724,
       "judge": 31.343283582089555,
       "standard": 62.318840579710155
      }
     },
     "sealed|score": {
      "cc": 45.6454723123812,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 68.67980276922125,
       "hard": 32.88780129611539,
       "judge": 46.813302889415894,
       "standard": 57.89190325094079
      }
     }
    }
   },
   "jevbench_score": 69.08416559129289,
   "key": "microsoft-decision-1",
   "last_measured_on": "2026-10-10",
   "listing": "ranked",
   "method": "v1.7",
   "model_pin": {
    "deployment": "decision-1",
    "model": "Microsoft-Decision-1",
    "provider": "Microsoft/Azure Foundry",
    "response_model": "microsoft-decision-1",
    "version": "1"
   },
   "noul_decisive": {
    "acc_among_decisive": 0.9340277777777778,
    "decisive_rate": 0.7721179624664879,
    "supported": true,
    "valid": 373
   },
   "previous": {
    "api_board_rank": 17,
    "axes": {
     "calibration": 84.39873071167106,
     "cost": 61.452075716621465,
     "intelligence": 57.16958510591244,
     "speed": 81.54517543213291
    },
    "board_release": "v1.6.43",
    "capability": 70.78415790879176,
    "capability_api_board_rank": 17,
    "capability_open_board_rank": null,
    "capability_rank": 41,
    "composite_ci95": {
     "A": [
      66.02436050159598,
      70.40957825472715
     ],
     "B": [
      62.32949160874876,
      68.09460025172818
     ],
     "C": [
      46.743685735358696,
      70.27466026550705
     ]
    },
    "jevbench_score": 69.11289175277149,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 29,
    "ranks": {
     "A": 29,
     "B": 38,
     "C": 40,
     "capability": 71
    },
    "scores": {
     "A": 69.11289175277149,
     "B": 66.34102888286164,
     "C": 62.74608663211473
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.45925578847527504,
     "p50_s_raw": 0.45925578847527504,
     "p95_s_adjusted": 1.5255548182874916,
     "p95_s_raw": 1.5255548182874916
    },
    "views": null
   },
   "rank": 83,
   "ranked": true,
   "ranks": {
    "A": 83,
    "B": 85,
    "C": 78,
    "capability": 91
   },
   "repo": "https://commandline.microsoft.com/microsoft-decision-1-model-foundry/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.08416559129289,
    "B": 66.31985230612224,
    "C": 62.72000677680797
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4787515588104725,
    "p50_s_raw": 0.4787515588104725,
    "p95_s_adjusted": 1.5182515516877173,
    "p95_s_raw": 1.5182515516877173
   },
   "status": {
    "answered_ok": 1479,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.38267975134342,
      86.54667450947431
     ],
     "I": [
      50.44641980227999,
      60.43490499481045
     ],
     "capability": [
      65.79924499904618,
      73.0808555865328
     ],
     "composite": {
      "A": [
       65.99814392729574,
       70.37976430138654
      ],
      "B": [
       62.310798268533894,
       68.07228955990982
      ],
      "C": [
       46.72503296526089,
       70.24484593257228
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": false,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 7.088097744249485,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.39873071167106,
     "capability": 70.78415790879176,
     "intelligence": 57.16958510591244
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7653333333333333,
      "calibration": 80.72765128361097,
      "cc": 69.61967049617368,
      "cc_by_type": {
       "choice": 69.61967049617368
      },
      "cc_tiered": 74.10245202356049,
      "cc_tiered_open": 82.42557001849185,
      "cc_tiered_sealed": 65.77933402862911,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.61967049617368
     },
     "noul": {
      "accuracy": 0.7173333333333334,
      "calibration": 88.35065717252257,
      "cc": 43.466666666666676,
      "cc_by_type": {
       "noul": 43.466666666666676
      },
      "cc_tiered": 44.546007779692246,
      "cc_tiered_open": 51.46376811594203,
      "cc_tiered_sealed": 37.62824744344246,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 43.466666666666676
     },
     "score": {
      "accuracy": 0.6933333333333334,
      "calibration": 84.11788367887968,
      "cc": 51.165347833026,
      "cc_by_type": {
       "score": 51.165347833026
      },
      "cc_tiered": 54.19533790280404,
      "cc_tiered_open": 62.74520349322689,
      "cc_tiered_sealed": 45.6454723123812,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 51.165347833026
     }
    },
    "raw_on_A": null,
    "run_sha256": "cadf99e94de455277604759b31028cfb14663f3b815555e5ea7909e0bb891c27",
    "set": "S+P"
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": true,
   "author": "Microsoft",
   "axes": {
    "calibration": 84.84355738336595,
    "cost": 62.19876038790379,
    "intelligence": 59.226098571054024,
    "speed": 88.66907977364731
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1108223199674765,
      "ece_score": 77.8355360065047,
      "mean_tvd": null,
      "n": 296,
      "n_ece_hard": 128,
      "n_tvd": 0,
      "score": 77.8355360065047,
      "score_before_time_limit": 77.8355360065047,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.22972972972972971,
      "brier": 0.08789565767934152,
      "ece": 0.06454285728875721,
      "n": 148,
      "score": 87.09142854224856,
      "score_before_time_limit": 87.09142854224856,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9194630872483222,
      "interval90_width": 0.1784116331096197,
      "n": 149,
      "nrps": 0.06306645857440242,
      "score": 87.1325461475575,
      "score_before_time_limit": 87.1325461475575,
      "time_limit_failed_share": 0,
      "top_ece": 0.09714130923722374
     }
    },
    "score": 84.01983689877025
   },
   "canonical_system_key": "microsoft-decision-1",
   "capability": 72.03482797720999,
   "composite_ci95": {
    "A": [
     59.165096184175106,
     74.53230995103505
    ],
    "B": [
     54.902945001197075,
     73.17537269212036
    ],
    "C": [
     41.08687235012159,
     74.53230995103505
    ]
   },
   "cost": {
    "basis": "PUBLISHED PRICING BASIS: USD 0.042/M input and USD 0/M output, basis of the live Microsoft-Decision-1 (Azure Foundry) row on https://benchmarkheaven.com/jev-models/api. Reported usage across all 600 decisions, including seven HTTP422 refusals retained in denominator; frozen parent all-item cost basis. Invoice, warmup and refusal charges unknown.",
    "common_basis": null,
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01819874
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Microsoft-Decision-1 (Azure Foundry) — A2 re-run",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted Azure Foundry API; latency measured from Sandy (Germany). Seven of 600 HTTP 422 responses scored wrong. Four self-hosted systems re-run on H200, versus H100 parents; wide equating interval. MeTask 31B and Torchcast not completed after four failed pod attempts; see measurement notes.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 67.00640482707561,
    "I_sealed": 46.29756060035449,
    "base": 56.65198271371505,
    "excess": 3.8882152491360813,
    "gap": 20.708844226721126,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 82.78833309663467,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 73.57659641396666,
       "judge": 84.17930125247197,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 54.94202898550725,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 63.288852399084945,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.860607370724196,
       "hard": 67.50339329867514,
       "judge": 74.36852911363773,
       "standard": 43.45437804225577
      }
     },
     "sealed|choice": {
      "cc": 63.30695607434754,
      "n": {
       "easy": 15,
       "hard": 64,
       "judge": 35,
       "standard": 36
      },
      "tiers": {
       "easy": 35.48387096774193,
       "hard": 52.901023890784984,
       "judge": 76.86375321336762,
       "standard": 77.69516728624535
      }
     },
     "sealed|noul": {
      "cc": 23.62820512820513,
      "n": {
       "easy": 13,
       "hard": 32,
       "judge": 15,
       "standard": 15
      },
      "tiers": {
       "easy": 84.61538461538463,
       "hard": 6.25,
       "judge": -6.666666666666665,
       "standard": 73.33333333333334
      }
     },
     "sealed|score": {
      "cc": 51.95752059851082,
      "n": {
       "easy": 9,
       "hard": 33,
       "judge": 20,
       "standard": 13
      },
      "tiers": {
       "easy": 76.44302715293294,
       "hard": 49.562049785657535,
       "judge": 44.71869741001873,
       "standard": 55.363943729744456
      }
     }
    }
   },
   "jevbench_score": 71.39513496315271,
   "key": "microsoft-decision-1-a2-20261012",
   "last_measured_on": "2026-10-12",
   "listing": "preliminary",
   "measured_in": "v1.6.14",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "not_ranked_because": "A2 supplement measurement with an incomplete equating pool (four of six); excluded from every ranking.",
   "planned_weight_denominator": 1,
   "preliminary": {
    "equating_pool_size": 4,
    "equating_pool_target": 6,
    "label": "PRELIMINARY — API supplement A2 (300 sealed + 300 public), not the S1200 full set; equating pool 4 of 6",
    "measured_on": "2026-10-12",
    "n_items": 600,
    "n_ok": 593,
    "scope": "API supplement A2"
   },
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.15566582345811,
     "cost": 62.19876038790379,
     "intelligence": 59.226098571054024,
     "speed": 88.69804989142763
    },
    "board_release": "v1.6.43",
    "capability": 72.19088219725606,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      60.33591615493133,
      74.53742586189195
     ],
     "B": [
      56.31152174249009,
      73.1804420130518
     ],
     "C": [
      41.89994177425786,
      74.53742586189195
     ]
    },
    "jevbench_score": 71.4549280926844,
    "measured_in": "v1.6.14",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": 71.4549280926844,
     "B": 68.62118813181577,
     "C": 69.62351351359915
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.3374342759998399,
     "p50_s_raw": 0.3374342759998399,
     "p95_s_adjusted": 0.3999501053016501,
     "p95_s_raw": 0.3999501053016501
    },
    "views": {
     "Balanced 33:33:33": 67.81784570375109,
     "Emphasis on Accuracy 60:20:20": 64.0984217053166,
     "Emphasis on Cost 20:20:60": 65.4526287582965,
     "Emphasis on Speed 20:60:20": 74.86761060072513,
     "Intelligence only": 59.22609857105403
    }
   },
   "provider": "microsoft",
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://commandline.microsoft.com/microsoft-decision-1-model-foundry/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 71.39513496315271,
    "B": 68.57706385643915,
    "C": 69.5652529027065
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.33810210000046936,
    "p50_s_raw": 0.33810210000046936,
    "p95_s_adjusted": 0.4018316626976227,
    "p95_s_raw": 0.4018316626976227
   },
   "status": {
    "answered_ok": 593,
    "missing": 0,
    "rows": 600,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": true,
    "api_subset_tag": "A2",
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 100
      },
      "calibration_gen": {
       "accuracy": 0.7692307692307693,
       "cc": 63.29111148533698,
       "cc_by_type": {
        "choice": 60.8695652173913,
        "noul": 75,
        "score": 54.003769238619604
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 63.29111148533698
      },
      "composite_gen": {
       "accuracy": 0.7741935483870968,
       "cc": 55.04557875346427,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 45.136736260392794
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 10,
        "noul": 10,
        "score": 11
       },
       "score": 55.04557875346427
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.45663234640188,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.36989703920564
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 6
       },
       "score": 99.45663234640188
      },
      "jaggedness": {
       "accuracy": 0.6,
       "cc": 36.18096361245757,
       "cc_by_type": {
        "choice": 72.78911564625851,
        "noul": -28.57142857142857,
        "score": 64.32520376254277
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 20,
        "noul": 14,
        "score": 11
       },
       "score": 36.18096361245757
      },
      "llm_router": {
       "accuracy": 0.8155339805825242,
       "cc": 77.29773373392734,
       "cc_by_type": {
        "choice": 66.38655462184875,
        "noul": 100,
        "score": 65.50664657993329
       },
       "low_n": false,
       "n": 103,
       "n_by_type": {
        "choice": 50,
        "noul": 30,
        "score": 23
       },
       "score": 77.29773373392734
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.3825136612022,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 1
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.7405660377358491,
       "cc": 56.84729346177679,
       "cc_by_type": {
        "choice": 67.37084291989123,
        "noul": 39.99999999999999,
        "score": 63.17103746543915
       },
       "low_n": false,
       "n": 212,
       "n_by_type": {
        "choice": 107,
        "noul": 40,
        "score": 65
       },
       "score": 56.84729346177679
      },
      "negation_pair_gen": {
       "accuracy": 0.8,
       "cc": 60,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 10,
        "noul": 10
       },
       "score": 60
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 21
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 1,
       "cc": 97.86434741302423,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 93.59304223907267
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 4
       },
       "score": 97.86434741302423
      },
      "usecase": {
       "accuracy": 0.7058823529411765,
       "cc": 37.19296114334664,
       "cc_by_type": {
        "choice": 67.7891654465593,
        "noul": 10.000000000000009,
        "score": 33.78971798348063
       },
       "low_n": false,
       "n": 85,
       "n_by_type": {
        "choice": 44,
        "noul": 20,
        "score": 21
       },
       "score": 37.19296114334664
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8888888888888888,
       "cc": 92.43674893442142,
       "cc_by_type": {
        "choice": 77.77777777777779,
        "noul": 100,
        "score": 99.53246902548645
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 1
       },
       "score": 92.43674893442142
      },
      "cs": {
       "accuracy": 0.625,
       "cc": 66.1654135338346,
       "cc_by_type": {
        "choice": 32.330827067669176,
        "noul": 100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 6,
        "noul": 2
       },
       "score": 66.1654135338346
      },
      "da": {
       "accuracy": 0.6666666666666666,
       "cc": 29.010593471104,
       "cc_by_type": {
        "choice": 80.83067092651757,
        "noul": 0,
        "score": 6.2011094867944205
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 1
       },
       "score": 29.010593471104
      },
      "de": {
       "accuracy": 0.9230769230769231,
       "cc": 90.62759668123668,
       "cc_by_type": {
        "choice": 74.64788732394366,
        "noul": 100,
        "score": 97.23490271976637
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 4
       },
       "score": 90.62759668123668
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": -21.42857142857143,
       "cc_by_type": {
        "choice": 57.14285714285714,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 3,
        "noul": 3
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.7753424657534247,
       "cc": 58.59078985584421,
       "cc_by_type": {
        "choice": 76.21638426861493,
        "noul": 42.574257425742566,
        "score": 56.981727873175124
       },
       "low_n": false,
       "n": 365,
       "n_by_type": {
        "choice": 185,
        "noul": 101,
        "score": 79
       },
       "score": 58.59078985584421
      },
      "es": {
       "accuracy": 0.9166666666666666,
       "cc": 70.0797903785218,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 10.239371135565412
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 1
       },
       "score": 70.0797903785218
      },
      "fi": {
       "accuracy": 1,
       "cc": 94.53482291929028,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 83.60446875787086
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 2,
        "noul": 2,
        "score": 6
       },
       "score": 94.53482291929028
      },
      "fr": {
       "accuracy": 0.75,
       "cc": 83.59037174371464,
       "cc_by_type": {
        "choice": 51.99999999999999,
        "noul": 100,
        "score": 98.77111523114391
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 1
       },
       "score": 83.59037174371464
      },
      "hi": {
       "accuracy": 1,
       "cc": 92.81199164662956,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 78.43597493988868
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 1
       },
       "score": 92.81199164662956
      },
      "id": {
       "accuracy": 0.7272727272727273,
       "cc": 70.7560754894387,
       "cc_by_type": {
        "choice": 73.09417040358746,
        "score": 68.41798057528993
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "score": 6
       },
       "score": 70.7560754894387
      },
      "it": {
       "accuracy": 0.7272727272727273,
       "cc": 80.04349357771646,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 40.13048073314939
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 2,
        "noul": 2,
        "score": 7
       },
       "score": 80.04349357771646
      },
      "ja": {
       "accuracy": 0.6363636363636364,
       "cc": 47.61381032028706,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 50,
        "score": 91.20208669856609
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 4,
        "score": 3
       },
       "score": 47.61381032028706
      },
      "ko": {
       "accuracy": 0.8181818181818182,
       "cc": 77.78079668122632,
       "cc_by_type": {
        "choice": 100,
        "score": 55.561593362452655
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 3,
        "score": 8
       },
       "score": 77.78079668122632
      },
      "mixed": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 11,
        "noul": 2
       },
       "score": 100
      },
      "nl": {
       "accuracy": 0.5714285714285714,
       "cc": 1.7832893135850607,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 5.349867940755182
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 1
       },
       "score": 1.7832893135850607
      },
      "no": {
       "accuracy": 0.6363636363636364,
       "cc": 39.253029666918,
       "cc_by_type": {
        "choice": 14.893617021276594,
        "noul": 33.33333333333333,
        "score": 69.53213864614409
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 3,
        "noul": 3,
        "score": 5
       },
       "score": 39.253029666918
      },
      "pl": {
       "accuracy": 0.7,
       "cc": 52.957577540994386,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": 50,
        "score": 40.901202373872835
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 8
       },
       "score": 52.957577540994386
      },
      "pt": {
       "accuracy": 0.6923076923076923,
       "cc": 74.25511482979587,
       "cc_by_type": {
        "choice": 26.530612244897956,
        "noul": 100,
        "score": 96.23473224448966
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 2
       },
       "score": 74.25511482979587
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 3.3459357774096503,
       "cc_by_type": {
        "choice": 33.70165745856353,
        "noul": -100,
        "score": 76.33614987366542
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 6
       },
       "score": 3.3459357774096503
      },
      "tr": {
       "accuracy": 0.7142857142857143,
       "cc": 77.61194029850746,
       "cc_by_type": {
        "choice": 55.22388059701492,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 77.61194029850746
      },
      "uk": {
       "accuracy": 0.5384615384615384,
       "cc": -2.908915160021811,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 35.71769896437901
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 3,
        "noul": 1,
        "score": 9
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.8333333333333334,
       "cc": 89.63050869746054,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 100,
        "score": 98.30329079826399
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 1
       },
       "score": 89.63050869746054
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7933333333333333,
       "cc": 73.01414581066375,
       "cc_by_type": {
        "choice": 73.01414581066375
       },
       "low_n": false,
       "n": 300,
       "n_by_type": {
        "choice": 300
       },
       "score": 73.01414581066375
      },
      "noul": {
       "accuracy": 0.7266666666666667,
       "cc": 45.333333333333336,
       "cc_by_type": {
        "noul": 45.333333333333336
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "noul": 150
       },
       "score": 45.333333333333336
      },
      "score": {
       "accuracy": 0.7466666666666667,
       "cc": 59.17272659737858,
       "cc_by_type": {
        "score": 59.17272659737858
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "score": 150
       },
       "score": 59.17272659737858
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6153846153846154,
       "cc": 22.655509813036304,
       "cc_by_type": {
        "choice": 73.7991266375546,
        "noul": -33.333333333333336,
        "score": 27.50073613488765
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 5
       },
       "score": 22.655509813036304
      },
      "code_linting": {
       "accuracy": 0.5555555555555556,
       "cc": 30.103576043627953,
       "cc_by_type": {
        "choice": 56.83453237410072,
        "noul": 0,
        "score": 33.47619575678314
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 30.103576043627953
      },
      "customer_support": {
       "accuracy": 0.875,
       "cc": 62.22727451527351,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 86.68182354582052
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 1
       },
       "score": 62.22727451527351
      },
      "demand_forecasting": {
       "accuracy": 0.2857142857142857,
       "cc": -25.73258663586017,
       "cc_by_type": {
        "choice": 16.89750692520776,
        "noul": -33.333333333333336,
        "score": -60.76193349945493
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6875,
       "cc": 37.128494536040996,
       "cc_by_type": {
        "choice": 51.61290322580645,
        "noul": -33.333333333333336,
        "score": 93.10591371564988
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 37.128494536040996
      },
      "feature_extraction": {
       "accuracy": 0.875,
       "cc": 79.10347860874892,
       "cc_by_type": {
        "choice": 85.94847775175644,
        "noul": 100,
        "score": 51.361958074490275
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 1,
        "score": 6
       },
       "score": 79.10347860874892
      },
      "financial_crime": {
       "accuracy": 0.8666666666666667,
       "cc": 86.22232602398712,
       "cc_by_type": {
        "choice": 80.70739549839227,
        "noul": 100,
        "score": 77.95958257356911
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 7,
        "noul": 2,
        "score": 6
       },
       "score": 86.22232602398712
      },
      "gaming": {
       "accuracy": 0.5333333333333333,
       "cc": 12.129649732040141,
       "cc_by_type": {
        "choice": 45.454545454545446,
        "noul": -33.333333333333336,
        "score": 24.267737074908315
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 7
       },
       "score": 12.129649732040141
      },
      "insurance_claims": {
       "accuracy": 0.5454545454545454,
       "cc": 11.15157429187248,
       "cc_by_type": {
        "choice": 43.573667711598745,
        "noul": 0,
        "score": -10.118944835981303
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 7,
        "noul": 2,
        "score": 2
       },
       "score": 11.15157429187248
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 65.92165664860643,
       "cc_by_type": {
        "choice": 56.83453237410071,
        "noul": 50,
        "score": 90.93043757171858
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 65.92165664860643
      },
      "lead_generation": {
       "accuracy": 0.5833333333333334,
       "cc": 32.27917770928797,
       "cc_by_type": {
        "choice": 52.56916996047431,
        "noul": 0,
        "score": 44.2683631673896
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 2
       },
       "score": 32.27917770928797
      },
      "legal_compliance": {
       "accuracy": 1,
       "cc": 94.12770453740406,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 82.38311361221218
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 4
       },
       "score": 94.12770453740406
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 87.81390014088669,
       "cc_by_type": {
        "choice": 78.14207650273225,
        "noul": 100,
        "score": 85.29962391992782
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 12,
        "noul": 1,
        "score": 5
       },
       "score": 87.81390014088669
      },
      "model_routing": {
       "accuracy": 0.8588235294117647,
       "cc": 81.49062517209573,
       "cc_by_type": {
        "choice": 79.13561847988078,
        "noul": 100,
        "score": 65.33625703640644
       },
       "low_n": false,
       "n": 85,
       "n_by_type": {
        "choice": 40,
        "noul": 24,
        "score": 21
       },
       "score": 81.49062517209573
      },
      "moderation_trust_safety": {
       "accuracy": 0.5833333333333334,
       "cc": 1.4715857588347916,
       "cc_by_type": {
        "choice": 61.29032258064516,
        "noul": -100,
        "score": 43.124434695859215
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 7,
        "noul": 1,
        "score": 4
       },
       "score": 1.4715857588347916
      },
      "other": {
       "accuracy": 0.4,
       "cc": 15.494864573175938,
       "cc_by_type": {
        "choice": 9.774436090225558,
        "score": 21.215293056126317
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 3,
        "score": 2
       },
       "score": 15.494864573175938
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 30.434646255259782,
       "cc_by_type": {
        "choice": 49.999999999999986,
        "noul": 0,
        "score": 41.30393876577936
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 4
       },
       "score": 30.434646255259782
      },
      "risk_assessment": {
       "accuracy": 0.75,
       "cc": 10.1937605340201,
       "cc_by_type": {
        "choice": 52.94117647058823,
        "noul": -100,
        "score": 77.64010513147205
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 5
       },
       "score": 10.1937605340201
      },
      "scientific_discovery": {
       "accuracy": 0.5714285714285714,
       "cc": 27.722363031181995,
       "cc_by_type": {
        "choice": 63.07692307692309,
        "noul": 0,
        "score": 20.09016601662289
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 7,
        "noul": 2,
        "score": 5
       },
       "score": 27.722363031181995
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.185292235786,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 94.55587670735798
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 4
       },
       "score": 98.185292235786
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.44693825231155,
      91.6174058639458
     ],
     "I": [
      47.46930943328842,
      68.3083574651847
     ],
     "capability": [
      64.10224318317934,
      77.56893471291383
     ],
     "composite": {
      "A": [
       59.165096184175106,
       74.53230995103505
      ],
      "B": [
       54.902945001197075,
       73.17537269212036
      ],
      "C": [
       41.08687235012159,
       74.53230995103505
      ]
     },
     "seed": 22
    },
    "ci95_basis": "bootstrap over the API system's items (n = A sealed items) AND over the equating offset (pool systems resampled with replacement, items resampled; B_offset = 200)",
    "ci95_n": 300,
    "complete": true,
    "equated": {
     "calibration": 85.15566582345811,
     "capability": 72.19088219725606,
     "intelligence": 59.226098571054024
    },
    "equating_flag": null,
    "equating_offset": {
     "C": 0.8237204845956931,
     "I": 2.574115857338972
    },
    "full_set_api": false,
    "g_med_used": 16.820628977585045,
    "lane": "api",
    "metric_basis": "A2 u P (sealed side = supplementary API subset A2, n=300)",
    "n_items": 600,
    "n_public": 300,
    "n_sealed_side": 300,
    "offset_ci95": {
     "C": [
      -2.9350424539001043,
      9.397802125135494
     ],
     "I": [
      -4.786775030874768,
      10.076945418608098
     ]
    },
    "on_S": null,
    "per_type": {
     "choice": {
      "accuracy": 0.7933333333333333,
      "calibration": 77.8355360065047,
      "cc": 73.01414581066375,
      "cc_by_type": {
       "choice": 73.01414581066375
      },
      "cc_tiered": 73.0476445854911,
      "cc_tiered_open": 82.78833309663467,
      "cc_tiered_sealed": 63.30695607434754,
      "low_n": false,
      "n": 300,
      "n_by_type": {
       "choice": 300
      },
      "score": 73.01414581066375
     },
     "noul": {
      "accuracy": 0.7266666666666667,
      "calibration": 87.09142854224856,
      "cc": 45.333333333333336,
      "cc_by_type": {
       "noul": 45.333333333333336
      },
      "cc_tiered": 39.285117056856194,
      "cc_tiered_open": 54.94202898550725,
      "cc_tiered_sealed": 23.62820512820513,
      "low_n": false,
      "n": 150,
      "n_by_type": {
       "noul": 150
      },
      "score": 45.333333333333336
     },
     "score": {
      "accuracy": 0.7466666666666667,
      "calibration": 87.1325461475575,
      "cc": 59.17272659737858,
      "cc_by_type": {
       "score": 59.17272659737858
      },
      "cc_tiered": 57.62318649879788,
      "cc_tiered_open": 63.288852399084945,
      "cc_tiered_sealed": 51.95752059851082,
      "low_n": false,
      "n": 150,
      "n_by_type": {
       "score": 150
      },
      "score": 59.17272659737858
     }
    },
    "raw_on_A": {
     "calibration": 84.01983689877025,
     "capability": 70.33590980624265,
     "intelligence": 56.65198271371505
    },
    "set": "A2+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 300
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 150
    },
    "score": {
     "invalid_rate": 0.00666666666666671,
     "n": 150
    }
   },
   "views": {
    "Balanced 33:33:33": 67.81219900588276,
    "Emphasis on Accuracy 60:20:20": 64.09539501387944,
    "Emphasis on Cost 20:20:60": 65.44947282925277,
    "Emphasis on Speed 20:60:20": 74.85522459350092,
    "Intelligence only": 59.22609857105403
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Microsoft",
   "axes": {
    "calibration": 92.50018348092215,
    "cost": 66.40926993775184,
    "intelligence": 67.11026330364345,
    "speed": 89.34376153841384
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.043287533474402395,
      "ece_score": 91.34249330511952,
      "mean_tvd": null,
      "n": 742,
      "n_ece_hard": 349,
      "n_tvd": 0,
      "score": 91.34249330511952,
      "score_before_time_limit": 91.34249330511952,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.17112299465240643,
      "brier": 0.07030502001380332,
      "ece": 0.03975394204046494,
      "n": 374,
      "score": 92.04921159190701,
      "score_before_time_limit": 92.04921159190701,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9490616621983914,
      "interval90_width": 0.13404825737265422,
      "n": 373,
      "nrps": 0.03751639673353231,
      "score": 94.10884554573991,
      "score_before_time_limit": 94.10884554573991,
      "time_limit_failed_share": 0,
      "top_ece": 0.04015334617583459
     }
    },
    "score": 92.50018348092215
   },
   "capability": 79.8052233922828,
   "composite_ci95": {
    "A": [
     74.34055361699605,
     78.09697479966553
    ],
    "B": [
     71.27669991591259,
     76.65874858081276
    ],
    "C": [
     74.34055361699605,
     78.09697479966553
    ]
   },
   "cost": {
    "basis": "Advertised Azure Foundry tariff USD 0.042 per million input tokens, output free (same basis as the first Microsoft-Decision-1 row); provider-reported usage. COST FIGURES: USD 0.013173 per 1,000 decisions, 0.20x the official cost cap USD 0.064595. Cost on the 1,489 items every row on this draw answered (same-items rule); all-items cost USD 0.013138 per 1,000 decisions. Over all n = 1,493 decisions: 467,021 input and 1,489 output tokens, the 4 refused requests in the denominator. Billed invoice unknown.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.013137898191560617
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.013173191403626596
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1493,
    "treatment": "share"
   },
   "disclosure": "SECOND MEASUREMENT (paid evaluation, public) of Microsoft-Decision-1 after a weights update reported by Microsoft. The served model id (microsoft-decision-1) is identical to the first measurement; we cannot verify a weights change behind an API. The first measurement (10 Oct 2026, another draw) stays listed unchanged as Microsoft-Decision-1 (Azure Foundry). ITEMS: n = 1,493 of the draw's 1,500 items (1,193 sealed + 300 public): 7 sealed items are excluded because this provider had seen them in an earlier evaluation, so every score and the speed are computed on those 1,493 items; the cost uses the 1,489 items every row on this draw answered (same-items rule). DRAW: measured on the 27 Oct 2026 regular draw (fresh sealed items; not yet equated to the reference scale, so its numbers are shown as published, offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own 1,493 items. LATENCY: p50 0.319 s, p95 0.364 s, maximum 0.605 s.",
   "display": "Microsoft-Decision-1 (updated weights, Oct 2026)",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-28",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.15",
    "results_sha256": "37d0d8f38f36a2d7dde8e3317199a72604417cf85748bc086e865d362a82b897",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted Azure AI Foundry deployment (Sweden Central), native API; latency measured from our server in Germany. 1,489 of 1,493 requests answered (HTTP 200); 4 were refused by the server because the prompt exceeded its 32,768-token context limit and are scored wrong; no decision took longer than 10 s.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 67.6831167669121,
    "I_sealed": 66.5374098403748,
    "base": 67.11026330364345,
    "excess": -1.4276807159009572,
    "gap": 1.1457069265372866,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 82.75192850961238,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 83.01352626612143,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 60.00759144237406,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 65.21739130434783,
       "judge": 30.000000000000004,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 60.28983034874987,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.13598476951736,
       "hard": 68.95389995873877,
       "judge": 69.16624778013274,
       "standard": 29.723987771314007
      }
     },
     "sealed|choice": {
      "cc": 78.0903732678344,
      "n": {
       "easy": 73,
       "hard": 284,
       "judge": 132,
       "standard": 106
      },
      "tiers": {
       "easy": 98.03471994759252,
       "hard": 67.420814479638,
       "judge": 80.8680248007086,
       "standard": 85.2908402050368
      }
     },
     "sealed|noul": {
      "cc": 47.08126821444638,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 9.756097560975618,
       "judge": 73.33333333333334,
       "standard": 62.7906976744186
      }
     },
     "sealed|score": {
      "cc": 74.44058803884367,
      "n": {
       "easy": 68,
       "hard": 111,
       "judge": 55,
       "standard": 64
      },
      "tiers": {
       "easy": 96.10011398522495,
       "hard": 57.66698582872063,
       "judge": 81.25918054104014,
       "standard": 86.93014073260439
      }
     }
    }
   },
   "jevbench_score": 76.97858754817413,
   "key": "microsoft-decision-1-r2",
   "last_measured_on": "2026-10-28",
   "listing": "unranked",
   "measured_in": "v1.7.15",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": {
    "deployment": "Decision-1",
    "model": "Microsoft-Decision-1",
    "provider": "Microsoft/Azure Foundry",
    "response_model": "microsoft-decision-1",
    "version": "updated weights, Oct 2026 (per Microsoft; served model id unchanged)"
   },
   "new_in": "v1.7.15",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale: its first anchor runs used models weaker than this row, and stronger anchors are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.9548387096774194,
    "decisive": 310,
    "decisive_rate": 0.8288770053475936,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "provider": "microsoft",
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://commandline.microsoft.com/microsoft-decision-1-model-foundry/",
   "scores": {
    "A": 76.97858754817413,
    "B": 74.7793794772783,
    "C": 76.97858754817413
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1493,
    "p50_s_adjusted": 0.31938244198681787,
    "p50_s_raw": 0.31938244198681787,
    "p95_s_adjusted": 0.3641772473813034,
    "p95_s_raw": 0.3641772473813034
   },
   "status": {
    "answered_ok": 1489,
    "missing": 0,
    "rows": 1493,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.75,
       "cc": 66.66666666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 66.66666666666666
      },
      "calibration_gen": {
       "accuracy": 0.6923076923076923,
       "cc": 62.36716220109435,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 100,
        "score": 13.075512577309013
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 6
       },
       "score": 62.36716220109435
      },
      "composite_gen": {
       "accuracy": 0.7894736842105263,
       "cc": 61.01781895173777,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 63.053456855213305
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 61.01781895173777
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.69547725058605,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.08643175175811
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 99.69547725058605
      },
      "jaggedness": {
       "accuracy": 0.7619047619047619,
       "cc": 42.03615727900286,
       "cc_by_type": {
        "choice": 87.65432098765433,
        "noul": -33.333333333333336,
        "score": 71.7874841826876
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 7
       },
       "score": 42.03615727900286
      },
      "llm_router": {
       "accuracy": 0.9101123595505618,
       "cc": 84.4445887387011,
       "cc_by_type": {
        "choice": 89.18594172424153,
        "noul": 100,
        "score": 64.14782449186177
       },
       "low_n": false,
       "n": 89,
       "n_by_type": {
        "choice": 44,
        "noul": 23,
        "score": 22
       },
       "score": 84.4445887387011
      },
      "long_state": {
       "accuracy": 0,
       "cc": -65.21739130434783,
       "cc_by_type": {
        "choice": -30.43478260869565,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 100
      },
      "multilingual": {
       "accuracy": 0.8295331161780674,
       "cc": 71.86054863813122,
       "cc_by_type": {
        "choice": 78.01250269454626,
        "noul": 58.19672131147542,
        "score": 79.37242190837199
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 71.86054863813122
      },
      "negation_pair_gen": {
       "accuracy": 0.9545454545454546,
       "cc": 91.66666666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 83.33333333333333
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 91.66666666666666
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 24
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.9642857142857143,
       "cc": 90.86569948935572,
       "cc_by_type": {
        "choice": 100,
        "noul": 80,
        "score": 92.59709846806716
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 90.86569948935572
      },
      "usecase": {
       "accuracy": 0.7876923076923077,
       "cc": 61.50305722121033,
       "cc_by_type": {
        "choice": 72.2830051057622,
        "noul": 38.46153846153846,
        "score": 73.76462809633033
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 61.50305722121033
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8571428571428571,
       "cc": 79.63050197791061,
       "cc_by_type": {
        "choice": 72.6027397260274,
        "noul": 66.66666666666667,
        "score": 99.62209954103777
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 79.63050197791061
      },
      "cs": {
       "accuracy": 0.7837837837837838,
       "cc": 58.237721506801485,
       "cc_by_type": {
        "choice": 75.93582887700535,
        "noul": 33.33333333333333,
        "score": 65.4440023100658
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 58.237721506801485
      },
      "da": {
       "accuracy": 0.7222222222222222,
       "cc": 45.09533596660589,
       "cc_by_type": {
        "choice": 77.63975155279502,
        "noul": 27.27272727272727,
        "score": 30.37352907429536
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 45.09533596660589
      },
      "de": {
       "accuracy": 0.8541666666666666,
       "cc": 80.38266499447833,
       "cc_by_type": {
        "choice": 81.47671726267045,
        "noul": 75,
        "score": 84.67127772076455
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 80.38266499447833
      },
      "el": {
       "accuracy": 0.7142857142857143,
       "cc": 52.59751967275878,
       "cc_by_type": {
        "choice": 65.38461538461539,
        "noul": 19.999999999999996,
        "score": 72.40794363366095
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 52.59751967275878
      },
      "en": {
       "accuracy": 0.8284671532846716,
       "cc": 69.13515279324577,
       "cc_by_type": {
        "choice": 79.87555901224965,
        "noul": 54.0983606557377,
        "score": 73.43153871174994
       },
       "low_n": false,
       "n": 548,
       "n_by_type": {
        "choice": 306,
        "noul": 122,
        "score": 120
       },
       "score": 69.13515279324577
      },
      "es": {
       "accuracy": 0.8918918918918919,
       "cc": 78.06994940384045,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 55.55555555555556,
        "score": 78.65429265596586
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 78.06994940384045
      },
      "fi": {
       "accuracy": 0.8653846153846154,
       "cc": 80.12348480247375,
       "cc_by_type": {
        "choice": 72.57142857142857,
        "noul": 78.94736842105263,
        "score": 88.85165741494005
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 80.12348480247375
      },
      "fr": {
       "accuracy": 0.875,
       "cc": 85.99860639201161,
       "cc_by_type": {
        "choice": 79.59183673469389,
        "noul": 100,
        "score": 78.40398244134093
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 85.99860639201161
      },
      "hi": {
       "accuracy": 0.8648648648648649,
       "cc": 70.98354111938421,
       "cc_by_type": {
        "choice": 91.73553719008264,
        "noul": 33.33333333333333,
        "score": 87.88175283473669
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 70.98354111938421
      },
      "id": {
       "accuracy": 0.8780487804878049,
       "cc": 82.44363109834262,
       "cc_by_type": {
        "choice": 81.5668202764977,
        "noul": 83.33333333333333,
        "score": 82.43073968519683
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 82.44363109834262
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 75.64061426232247,
       "cc_by_type": {
        "choice": 92.01419698314109,
        "noul": 66.66666666666667,
        "score": 68.24097913715966
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 75.64061426232247
      },
      "ja": {
       "accuracy": 0.9090909090909091,
       "cc": 83.9650875167486,
       "cc_by_type": {
        "choice": 84.45595854922279,
        "noul": 81.81818181818181,
        "score": 85.62112218284116
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 83.9650875167486
      },
      "ko": {
       "accuracy": 0.813953488372093,
       "cc": 74.56823829100087,
       "cc_by_type": {
        "choice": 63.23529411764706,
        "noul": 80,
        "score": 80.46942075535557
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 74.56823829100087
      },
      "mixed": {
       "accuracy": 0.921875,
       "cc": 88.90529539345869,
       "cc_by_type": {
        "choice": 89.1989198919892,
        "noul": 89.99999999999999,
        "score": 87.51696628838684
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 88.90529539345869
      },
      "nl": {
       "accuracy": 0.8055555555555556,
       "cc": 52.066295624257975,
       "cc_by_type": {
        "choice": 85.84905660377359,
        "noul": 39.99999999999999,
        "score": 30.349830269000343
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 52.066295624257975
      },
      "no": {
       "accuracy": 0.7450980392156863,
       "cc": 60.32134770287596,
       "cc_by_type": {
        "choice": 63.85542168674698,
        "noul": 33.33333333333333,
        "score": 83.77528808854758
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 60.32134770287596
      },
      "pl": {
       "accuracy": 0.8245614035087719,
       "cc": 63.532302508293675,
       "cc_by_type": {
        "choice": 85.63065460351251,
        "noul": 64.70588235294117,
        "score": 40.26037056842734
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 63.532302508293675
      },
      "pt": {
       "accuracy": 0.723404255319149,
       "cc": 46.56624649809212,
       "cc_by_type": {
        "choice": 70.04991680532446,
        "noul": 0,
        "score": 69.64882268895192
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 46.56624649809212
      },
      "sv": {
       "accuracy": 0.7391304347826086,
       "cc": 56.275340157256146,
       "cc_by_type": {
        "choice": 53.07262569832402,
        "noul": 33.33333333333333,
        "score": 82.42006144011108
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 56.275340157256146
      },
      "tr": {
       "accuracy": 0.8260869565217391,
       "cc": 78.5949259667293,
       "cc_by_type": {
        "choice": 65.79247434435577,
        "noul": 80,
        "score": 89.99230355583217
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 78.5949259667293
      },
      "uk": {
       "accuracy": 0.9285714285714286,
       "cc": 84.90072885258873,
       "cc_by_type": {
        "choice": 83.56164383561644,
        "noul": 85.71428571428572,
        "score": 85.42625700786401
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 84.90072885258873
      },
      "zh": {
       "accuracy": 0.7804878048780488,
       "cc": 64.11305504013963,
       "cc_by_type": {
        "choice": 81.40495867768595,
        "noul": 50,
        "score": 60.93420644273293
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 64.11305504013963
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8456375838926175,
       "cc": 78.92636444154863,
       "cc_by_type": {
        "choice": 78.92636444154863
       },
       "low_n": false,
       "n": 745,
       "n_by_type": {
        "choice": 745
       },
       "score": 78.92636444154863
      },
      "noul": {
       "accuracy": 0.7893333333333333,
       "cc": 57.86666666666667,
       "cc_by_type": {
        "noul": 57.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 57.86666666666667
      },
      "score": {
       "accuracy": 0.839142091152815,
       "cc": 76.63215333353354,
       "cc_by_type": {
        "score": 76.63215333353354
       },
       "low_n": false,
       "n": 373,
       "n_by_type": {
        "score": 373
       },
       "score": 76.63215333353354
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6875,
       "cc": 50.56682379155412,
       "cc_by_type": {
        "choice": 55.52260934025203,
        "noul": 14.28571428571428,
        "score": 81.89214774869605
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 50.56682379155412
      },
      "code_linting": {
       "accuracy": 0.8695652173913043,
       "cc": 73.34717607484559,
       "cc_by_type": {
        "choice": 81.62890385793018,
        "noul": 66.66666666666667,
        "score": 71.7459576999399
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 73.34717607484559
      },
      "customer_support": {
       "accuracy": 0.8484848484848485,
       "cc": 70.04553403127171,
       "cc_by_type": {
        "choice": 85.239852398524,
        "noul": 33.33333333333333,
        "score": 91.56341636195783
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 70.04553403127171
      },
      "demand_forecasting": {
       "accuracy": 0.6212121212121212,
       "cc": 19.137559201546473,
       "cc_by_type": {
        "choice": 54.510993176648974,
        "noul": -53.84615384615385,
        "score": 56.74783827414429
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 19.137559201546473
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8387096774193549,
       "cc": 68.2015036770102,
       "cc_by_type": {
        "choice": 87.33286418015481,
        "noul": 41.176470588235304,
        "score": 76.09517626264048
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 68.2015036770102
      },
      "feature_extraction": {
       "accuracy": 0.8055555555555556,
       "cc": 70.07429463989408,
       "cc_by_type": {
        "choice": 69.84924623115577,
        "noul": 71.42857142857142,
        "score": 68.94506625995506
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 70.07429463989408
      },
      "financial_crime": {
       "accuracy": 0.7936507936507936,
       "cc": 47.59506984593074,
       "cc_by_type": {
        "choice": 84.05103668261565,
        "noul": -9.090909090909093,
        "score": 67.82508194608566
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 47.59506984593074
      },
      "gaming": {
       "accuracy": 0.8275862068965517,
       "cc": 69.19235835255368,
       "cc_by_type": {
        "choice": 80.87649402390439,
        "noul": 55.55555555555556,
        "score": 71.14502547820109
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 69.19235835255368
      },
      "insurance_claims": {
       "accuracy": 0.6451612903225806,
       "cc": 44.44771429838312,
       "cc_by_type": {
        "choice": 42.09354120267261,
        "noul": 29.41176470588236,
        "score": 61.8378369865944
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 44.44771429838312
      },
      "knowledge_graphs": {
       "accuracy": 0.7910447761194029,
       "cc": 68.28349865454777,
       "cc_by_type": {
        "choice": 77.51124437781112,
        "noul": 84.61538461538463,
        "score": 42.72386697044755
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 68.28349865454777
      },
      "lead_generation": {
       "accuracy": 0.8266666666666667,
       "cc": 74.27692934102033,
       "cc_by_type": {
        "choice": 66.14950634696756,
        "noul": 77.14285714285714,
        "score": 79.5384245332363
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 74.27692934102033
      },
      "legal_compliance": {
       "accuracy": 0.8392857142857143,
       "cc": 68.76798429109556,
       "cc_by_type": {
        "choice": 87.96389167502507,
        "noul": 42.85714285714286,
        "score": 75.48291834111875
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 68.76798429109556
      },
      "llm_guardrails": {
       "accuracy": 0.9577464788732394,
       "cc": 90.4783319603717,
       "cc_by_type": {
        "choice": 91.85336048879836,
        "noul": 92.3076923076923,
        "score": 87.27394308462445
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 90.4783319603717
      },
      "model_routing": {
       "accuracy": 0.8918918918918919,
       "cc": 82.70792730455777,
       "cc_by_type": {
        "choice": 85.97888177254633,
        "noul": 94.11764705882352,
        "score": 68.02725308230345
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 82.70792730455777
      },
      "moderation_trust_safety": {
       "accuracy": 0.9152542372881356,
       "cc": 82.47828693182674,
       "cc_by_type": {
        "choice": 93.4640522875817,
        "noul": 64.70588235294117,
        "score": 89.26492615495735
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 82.47828693182674
      },
      "other": {
       "accuracy": 0.75,
       "cc": 58.526608178705395,
       "cc_by_type": {
        "choice": 66.70366259711432,
        "noul": 33.33333333333333,
        "score": 75.54282860566853
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 58.526608178705395
      },
      "recruiting": {
       "accuracy": 0.8615384615384616,
       "cc": 80.16582023277427,
       "cc_by_type": {
        "choice": 74.04470079307859,
        "noul": 84.00000000000001,
        "score": 82.4527599052442
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 80.16582023277427
      },
      "risk_assessment": {
       "accuracy": 0.7291666666666666,
       "cc": 47.531368798803136,
       "cc_by_type": {
        "choice": 74.89539748953975,
        "noul": -7.692307692307687,
        "score": 75.39101659917733
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 47.531368798803136
      },
      "scientific_discovery": {
       "accuracy": 0.7857142857142857,
       "cc": 59.47394665472459,
       "cc_by_type": {
        "choice": 85.27607361963189,
        "noul": 19.999999999999996,
        "score": 73.1457663445419
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 59.47394665472459
      },
      "search_retrieval": {
       "accuracy": 0.9850746268656716,
       "cc": 94.83767892719463,
       "cc_by_type": {
        "choice": 100,
        "noul": 85.71428571428572,
        "score": 98.79875106729818
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 94.83767892719463
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.48661137990368,
      92.64828141801456
     ],
     "I": [
      61.21043983722205,
      71.38216112717504
     ],
     "capability": [
      75.11440308146514,
      81.43921372433407
     ],
     "composite": {
      "A": [
       74.34055361699605,
       78.09697479966553
      ],
      "B": [
       71.27669991591259,
       76.65874858081276
      ],
      "C": [
       74.34055361699605,
       78.09697479966553
      ]
     },
     "seed": 16
    },
    "ci95_n": 1193,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1493,
    "n_public": 300,
    "n_sealed_side": 1193,
    "on_S": {
     "calibration": 92.50018348092215,
     "capability": 79.8052233922828,
     "intelligence": 67.11026330364345
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8456375838926175,
      "calibration": 91.34249330511952,
      "cc": 78.92636444154863,
      "cc_by_type": {
       "choice": 78.92636444154863
      },
      "cc_tiered": 80.42115088872339,
      "cc_tiered_open": 82.75192850961238,
      "cc_tiered_sealed": 78.0903732678344,
      "low_n": false,
      "n": 745,
      "n_by_type": {
       "choice": 745
      },
      "score": 78.92636444154863
     },
     "noul": {
      "accuracy": 0.7893333333333333,
      "calibration": 92.04921159190701,
      "cc": 57.86666666666667,
      "cc_by_type": {
       "noul": 57.86666666666667
      },
      "cc_tiered": 53.544429828410216,
      "cc_tiered_open": 60.00759144237406,
      "cc_tiered_sealed": 47.08126821444638,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 57.86666666666667
     },
     "score": {
      "accuracy": 0.839142091152815,
      "calibration": 94.10884554573991,
      "cc": 76.63215333353354,
      "cc_by_type": {
       "score": 76.63215333353354
      },
      "cc_tiered": 67.36520919379677,
      "cc_tiered_open": 60.28983034874987,
      "cc_tiered_sealed": 74.44058803884367,
      "low_n": false,
      "n": 373,
      "n_by_type": {
       "score": 373
      },
      "score": 76.63215333353354
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.004026845637583931,
     "n": 745
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 373
    }
   },
   "views": {
    "Balanced 33:33:33": 72.90097585166698,
    "Emphasis on Accuracy 60:20:20": 70.46877315212157,
    "Emphasis on Cost 20:20:60": 70.15772440584237,
    "Emphasis on Speed 20:60:20": 78.69410707982621,
    "Intelligence only": 67.11026330364345
   }
  },
  {
   "adapter_id": "images/mirror/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bluusun",
   "axes": {
    "calibration": 51.61277239297377,
    "cost": 89.27300805786774,
    "intelligence": 0,
    "speed": 74.03935684279031
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.2204942558257705,
      "ece_score": 55.90114883484589,
      "mean_tvd": null,
      "n": 379,
      "n_ece_hard": 100,
      "n_tvd": 0,
      "score": 55.90114883484589,
      "score_before_time_limit": 55.90114883484589,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.07773851590106007,
      "brier": 0.3825718867287919,
      "ece": 0.37640864622953524,
      "n": 283,
      "score": 24.718270754092952,
      "score_before_time_limit": 24.718270754092952,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9471544715447154,
      "interval90_width": 0.8113143631436313,
      "n": 246,
      "nrps": 0.18235540291655192,
      "score": 74.21889758998248,
      "score_before_time_limit": 74.21889758998248,
      "time_limit_failed_share": 0,
      "top_ece": 0.1666333226418992
     }
    },
    "score": 51.61277239297377
   },
   "capability": 25.806386196486883,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0,
     9.853496598484777e-09
    ],
    "B": [
     0,
     6.1614500141756846e-09
    ],
    "C": [
     0,
     6.8427059711699815e-09
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0022780665024630543
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Mirror",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 0,
    "I_sealed": 0,
    "base": 0,
    "excess": -2.573387642438244,
    "gap": 0,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 0,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": -7.5810003145643154,
       "judge": -14.700065919578117,
       "standard": -5.4834658853076625
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0,
       "hard": -73.91304347826086,
       "judge": -19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 14.02621656810803,
       "hard": -86.48465684738163,
       "judge": -131.29938184642268,
       "standard": -61.876452149889175
      }
     },
     "sealed|choice": {
      "cc": 0,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 14.86216835414975,
       "hard": -16.063348416289617,
       "judge": -12.662942271880798,
       "standard": -6.417046434289924
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 12.00000000000001,
       "hard": -33.98058252427184,
       "judge": -15.789473684210531,
       "standard": -7.042253521126762
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -1.6525187057042423,
       "hard": -113.52957658615944,
       "judge": -73.84737475308705,
       "standard": -29.673128809498152
      }
     }
    }
   },
   "jevbench_score": 0,
   "key": "mirror",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 wrapper and Mirror release; upstream DeBERTa/model assets retain their own terms",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5977011494252874,
    "decisive_rate": 0.9222614840989399,
    "supported": true,
    "valid": 283
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 51.61277239297377,
     "cost": 89.27300805786774,
     "intelligence": 0,
     "speed": 73.47211485993975
    },
    "board_release": "v1.6.43",
    "capability": 25.806386196486883,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0,
      9.853474730694675e-09
     ],
     "B": [
      0,
      6.161443172813211e-09
     ],
     "C": [
      0,
      6.8426907852046335e-09
     ]
    },
    "jevbench_score": 0,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 196,
    "rank": 240,
    "ranks": {
     "A": 240,
     "B": 240,
     "C": 240,
     "capability": 233
    },
    "scores": {
     "A": 0,
     "B": 0,
     "C": 0
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.4516588259488343,
     "p50_s_raw": 0.6508294129744172,
     "p95_s_adjusted": 3.096876988979056,
     "p95_s_raw": 1.473438494489528
    },
    "views": {
     "Balanced 33:33:33": 0,
     "Emphasis on Accuracy 60:20:20": 0,
     "Emphasis on Cost 20:20:60": 0,
     "Emphasis on Speed 20:60:20": 0,
     "Intelligence only": 0
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 281,
   "ranked": true,
   "ranks": {
    "A": 281,
    "B": 281,
    "C": 281,
    "capability": 274
   },
   "repo": "https://github.com/Bluusun/Decision-API",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0,
    "B": 0,
    "C": 0
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.2716480298899113,
    "p50_s_raw": 0.5608240149449557,
    "p95_s_adjusted": 3.10239715129137,
    "p95_s_raw": 1.476198575645685
   },
   "status": {
    "answered_ok": 908,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Mirror DeBERTa-v3-large 436M classification-span scorer",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      42.56771416804497,
      56.74054715693854
     ],
     "I": [
      0,
      0.01760273906633102
     ],
     "capability": [
      21.283857084022486,
      28.426215016960548
     ],
     "composite": {
      "A": [
       0,
       9.853496598484777e-09
      ],
      "B": [
       0,
       6.1614500141756846e-09
      ],
      "C": [
       0,
       6.8427059711699815e-09
      ]
     },
     "seed": 68
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 51.61277239297377,
     "capability": 25.806386196486883,
     "intelligence": 0
    },
    "per_type": {
     "choice": {
      "accuracy": 0.156,
      "calibration": 55.90114883484589,
      "cc": -7.935545598363106,
      "cc_by_type": {
       "choice": -7.935545598363106
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0
     },
     "noul": {
      "accuracy": 0.416,
      "calibration": 24.718270754092952,
      "cc": -16.800000000000004,
      "cc_by_type": {
       "noul": -16.800000000000004
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.18666666666666668,
      "calibration": 74.21889758998248,
      "cc": -72.82982094179775,
      "cc_by_type": {
       "score": -72.82982094179775
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 48.60190522372855,
     "capability": 24.300952611864275,
     "intelligence": 0
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.4946666666666667,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.2453333333333333,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.344,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0,
    "Emphasis on Accuracy 60:20:20": 0,
    "Emphasis on Cost 20:20:60": 0,
    "Emphasis on Speed 20:60:20": 0,
    "Intelligence only": 0
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ksonny4 / @kubelpe (ksonny4@gmail.com)",
   "axes": {
    "calibration": 66.68251917899094,
    "cost": 70.36992715953406,
    "intelligence": 2.6107469083924393,
    "speed": 57.79924670989358
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.16197772093040966,
      "ece_score": 67.60445581391808,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 67.60445581391808
     },
     "noul": {
      "abstention_rate": 1,
      "brier": 0.28875559863871086,
      "ece": 0.20363870372723708,
      "n": 375,
      "score": 59.27225925455259
     },
     "score": {
      "interval90_coverage": 0.96,
      "interval90_width": 0.8991111111111114,
      "n": 375,
      "nrps": 0.1846962110408624,
      "score": 73.17084246850217,
      "top_ece": 0.17594346979454697
     }
    },
    "score": 66.68251917899094
   },
   "capability": 34.64663304369169,
   "composite_ci95": {
    "A": [
     0.0008967609092971513,
     0.12984410368517996
    ],
    "B": [
     0.0005711694021380616,
     0.08896822577663006
    ],
    "C": [
     0.000622750631456355,
     0.09016951644804166
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no public tariff. The same choice the published quyet-1-0-small-en row carries for the identical ModernBERT-base backbone: DeepInfra base-size encoders at USD 0.005 per 1M input and 0 output (deepinfra:thenlper/gte-base, BAAI/bge-base-en-v1.5, intfloat/e5-base-v2 and sentence-transformers/all-mpnet-base-v2 are all 5e-7 cents per input token in the frozen 25 Sep 2026 DeepInfra snapshot). Output tokens are a literal 0 reported by the author's server (NLI entailment read-out, nothing is generated). This row's own usage on draw v1.6-regular-20261010-a2: 2,916,019 input and 0 output tokens over 1,500 decisions. Estimated, not charged.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009720063333333334
   },
   "decision_limit": null,
   "display": "modernbert-jev v2",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Sandy, the evaluator's own shared CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2), NOT a dedicated machine: other jobs ran beside it and the 1-minute load average was logged every 60 s. The author's own CLI at github.com/KSonny4/modernbert-jev@b1866e4d, `modernbert-jev serve --model <pinned snapshot dir>` at its own defaults (1 ONNX Runtime thread, 127.0.0.1), inside a loopback-only network namespace with an outbound probe that failed as required, no credentials in the environment. int8 (unsigned) ONNX as shipped. Speed is a lower bound for this model on a dedicated core.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 3.8269990036680306,
    "I_sealed": 1.394494813116848,
    "base": 2.6107469083924393,
    "excess": -7.6407403907885305,
    "gap": 2.4325041905511826,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 11.480997011004092,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486504,
       "hard": -1.9188424032714584,
       "judge": 20.89650626235992,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 4.480029631602289,
       "hard": -0.07960342036494872,
       "judge": 1.7876323239060077,
       "standard": -15.095565734194793
      }
     },
     "sealed|choice": {
      "cc": 4.183484439350544,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 8.019605798310561,
       "hard": 2.055622732769036,
       "judge": 3.2012581807112888,
       "standard": 7.994486560992435
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 1.478176928885322,
       "hard": -3.5620793106463378,
       "judge": -5.974082989668261,
       "standard": -11.414183075878537
      }
     }
    }
   },
   "jevbench_score": 0.025388991293562355,
   "key": "modernbert-jev-v2",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (model card; the code README and pyproject.toml also declare Apache-2.0, without a separate LICENSE file)",
   "listing": "remeasure-pending",
   "measured_in": "v1.6.8",
   "measurement_date_status": "measured",
   "method": "v1.6",
   "model_pin": "ksonny4/modernbert-jev-v2@9708136f22a85b29fcf90d3960aa642d9eadcb1c",
   "not_ranked_because": "Re-measure pending under the 10 s rule: this row's per-decision times were recorded on a loaded CPU host, so they cannot be scored under the time limit. Shown with its v1.6 values until it is measured again.",
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive": 0,
    "decisive_rate": 0,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 66.68251917899094,
     "cost": 70.36992715953406,
     "intelligence": 2.6107469083924393,
     "speed": 57.79924670989358
    },
    "board_release": "v1.6.43",
    "capability": 34.64663304369169,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.0008967609092971513,
      0.12984410368517996
     ],
     "B": [
      0.0005711694021380616,
      0.08896822577663006
     ],
     "C": [
      0.000622750631456355,
      0.09016951644804166
     ]
    },
    "jevbench_score": 0.025388991293562355,
    "measured_in": "v1.6.8",
    "method": "v1.6",
    "open_board_rank": 185,
    "rank": 229,
    "ranks": {
     "A": 229,
     "B": 229,
     "C": 229,
     "capability": 217
    },
    "scores": {
     "A": 0.025388991293562355,
     "B": 0.016776345440377434,
     "C": 0.01763124395386275
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 5.349294321238995,
     "p50_s_raw": 2.5996471606194973,
     "p95_s_adjusted": 31.02978992164135,
     "p95_s_raw": 15.439894960820675
    },
    "views": {
     "Balanced 33:33:33": 0.01973059249456439,
     "Emphasis on Accuracy 60:20:20": 0.011546588753666046,
     "Emphasis on Cost 20:20:60": 0.030774419703851856,
     "Emphasis on Speed 20:60:20": 0.030350892509696595,
     "Intelligence only": 0.0071179397650143975
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/ksonny4/modernbert-jev-v2",
   "scores": {
    "A": 0.025388991293562355,
    "B": 0.016776345440377434,
    "C": 0.01763124395386275
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "n": 124,
    "p50_s_adjusted": 5.349294321238995,
    "p50_s_raw": 2.5996471606194973,
    "p95_s_adjusted": 31.02978992164135,
    "p95_s_raw": 15.439894960820675
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.13793103448275862,
       "cc": -43.39863396756374,
       "cc_by_type": {
        "choice": -8.333333333333334,
        "noul": -100,
        "score": -21.862568569357887
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.17857142857142858,
       "cc": -45.8159894104131,
       "cc_by_type": {
        "choice": 2.173913043478264,
        "noul": -100,
        "score": -39.621881274717595
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.22448979591836735,
       "cc": -32.536005477410086,
       "cc_by_type": {
        "choice": 3.7037037037037055,
        "noul": -100,
        "score": -1.3117201359339692
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.3333333333333333,
       "cc": -25.18773661803903,
       "cc_by_type": {
        "choice": -5.882352941176475,
        "noul": -100,
        "score": 30.319143087059388
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.14594594594594595,
       "cc": -46.20038451819038,
       "cc_by_type": {
        "choice": -12.552107457156087,
        "noul": -100,
        "score": -26.04904609741505
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.18181818181818182,
       "cc": -31.90735375118363,
       "cc_by_type": {
        "choice": 10.822108685555056,
        "noul": -100,
        "score": -6.544169939105937
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 0
      },
      "long_state": {
       "accuracy": 0.034482758620689655,
       "cc": -44.327454537857705,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -2.0299826611921645
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.1935483870967742,
       "cc": -36.52576519511609,
       "cc_by_type": {
        "choice": -1.5384615384615408,
        "noul": -100,
        "score": -8.038834046886723
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.20952380952380953,
       "cc": -28.34675764556054,
       "cc_by_type": {
        "choice": 6.8517689207344405,
        "noul": -100,
        "score": 8.107958142583938
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.14285714285714285,
       "cc": -56.58208630213584,
       "cc_by_type": {
        "choice": -4.761904761904764,
        "noul": -100,
        "score": -64.98435414450276
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.21052631578947367,
       "cc": 0.11098779134295962,
       "cc_by_type": {
        "choice": 0.11098779134295962
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 0.11098779134295962
      },
      "perturb_pair": {
       "accuracy": 0.2833333333333333,
       "cc": -18.717250201930284,
       "cc_by_type": {
        "choice": 17.431192660550455,
        "noul": -100,
        "score": 26.417056733658693
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.2489451476793249,
       "cc": -25.635511113946517,
       "cc_by_type": {
        "choice": 21.37213335902873,
        "noul": -100,
        "score": 1.7213332991317332
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.2727272727272727,
       "cc": -34.51061721771425,
       "cc_by_type": {
        "choice": -6.666666666666665,
        "noul": -100,
        "score": 3.134815013523906
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.125,
       "cc": -50.847457627118644,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": -100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.3333333333333333,
       "cc": -20.541065093618435,
       "cc_by_type": {
        "choice": 40.8284023668639,
        "noul": -100,
        "score": -2.451597647719206
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.2,
       "cc": -32.195675052815936,
       "cc_by_type": {
        "choice": 2.232951116475559,
        "noul": -100,
        "score": 1.1800237250766288
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 0
      },
      "el": {
       "accuracy": 0.16666666666666666,
       "cc": -50.44316644371644,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -100,
        "score": -54.94395716247462
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.19571045576407506,
       "cc": -34.73453804072174,
       "cc_by_type": {
        "choice": 4.206393718451972,
        "noul": -100,
        "score": -8.410007840617183
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.25,
       "cc": -18.257637042739912,
       "cc_by_type": {
        "choice": 21.45454545454546,
        "noul": -100,
        "score": 23.772543417234793
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.2222222222222222,
       "cc": -42.25352112676057,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": -100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.4,
       "cc": -12.310038757433736,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": -100,
        "score": -11.542246032526894
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.13333333333333333,
       "cc": -35.49650623401251,
       "cc_by_type": {
        "choice": -11.801242236024846,
        "noul": -100,
        "score": 5.3117235339873226
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.3333333333333333,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.29411764705882354,
       "cc": -22.98844453863286,
       "cc_by_type": {
        "choice": 23.83638928067701,
        "noul": -100,
        "score": 7.198277103424422
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.125,
       "cc": -40.139871804621805,
       "cc_by_type": {
        "choice": -13.821138211382111,
        "noul": -100,
        "score": -6.598477202483299
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.2727272727272727,
       "cc": -36.975867376244004,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": -12.56694639102709
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0,
       "cc": -40.20521661272176,
       "cc_by_type": {
        "choice": -31.868131868131872,
        "noul": -100,
        "score": 11.252482029966604
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.07692307692307693,
       "cc": -29.998198543905005,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": -100,
        "score": 10.283955900318398
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.2857142857142857,
       "cc": -32.78688524590164,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.2,
       "cc": -24.81588086706398,
       "cc_by_type": {
        "choice": 5.553126366418887,
        "noul": -100,
        "score": 19.999231032389165
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.2777777777777778,
       "cc": -9.14274565490507,
       "cc_by_type": {
        "choice": 63.81909547738693,
        "noul": -100,
        "score": 8.752667557897864
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0,
       "cc": -43.590934068488735,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": 2.5605311278671383
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.36363636363636365,
       "cc": -17.28662918547593,
       "cc_by_type": {
        "choice": 21.052631578947373,
        "noul": -100,
        "score": 27.087480864624837
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.2727272727272727,
       "cc": -23.219083224832847,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": -100,
        "score": 39.038402499414495
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0,
       "cc": -44.80276763421367,
       "cc_by_type": {
        "choice": -29.729729729729726,
        "noul": -100,
        "score": -4.678573172911271
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.268,
       "cc": 5.164968042839871,
       "cc_by_type": {
        "choice": 5.164968042839871
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 5.164968042839871
      },
      "noul": {
       "accuracy": 0,
       "cc": -100,
       "cc_by_type": {
        "noul": -100
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.2613333333333333,
       "cc": -4.528144639423148,
       "cc_by_type": {
        "score": -4.528144639423148
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.375,
       "cc": -20.45434194583145,
       "cc_by_type": {
        "choice": 30.807248764415156,
        "noul": -100,
        "score": 7.829725398090492
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": -23.589501736030243,
       "cc_by_type": {
        "choice": 31.26022913256956,
        "noul": -100,
        "score": -2.0287343406602876
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.15,
       "cc": -30.19926598996379,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100,
        "score": 6.176395578495719
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.15384615384615385,
       "cc": -40.108407064559685,
       "cc_by_type": {
        "choice": -8.43373493975904,
        "noul": -100,
        "score": -11.891486253920025
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2777777777777778,
       "cc": -17.392819381831114,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100,
        "score": 3.722162972519083
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.1724137931034483,
       "cc": -35.42851137104405,
       "cc_by_type": {
        "choice": -2.564102564102566,
        "noul": -100,
        "score": -3.721431549029597
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.19230769230769232,
       "cc": -26.23012292511098,
       "cc_by_type": {
        "choice": 8.695652173913043,
        "noul": -100,
        "score": 12.61397905075401
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.1724137931034483,
       "cc": -30.62771796722504,
       "cc_by_type": {
        "choice": 4.950495049504946,
        "noul": -100,
        "score": 3.1663510488199376
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.42857142857142855,
       "cc": 9.59819138604467,
       "cc_by_type": {
        "choice": 52.38095238095238,
        "noul": -100,
        "score": 76.41362177718163
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 9.59819138604467
      },
      "knowledge_graphs": {
       "accuracy": 0.13043478260869565,
       "cc": -38.28700352328199,
       "cc_by_type": {
        "choice": -0.7633587786259586,
        "noul": -100,
        "score": -14.09765179122
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.16,
       "cc": -32.661969492218304,
       "cc_by_type": {
        "choice": 2.4390243902439046,
        "noul": -100,
        "score": -0.4249328668988195
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.38461538461538464,
       "cc": -15.740475832931393,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": 18.352342993009096
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.3448275862068966,
       "cc": -22.60830157830547,
       "cc_by_type": {
        "choice": 26.60550458715596,
        "noul": -100,
        "score": 5.569590677927638
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.21333333333333335,
       "cc": -26.190070048541028,
       "cc_by_type": {
        "choice": 19.592602519431804,
        "noul": -100,
        "score": 1.8371873349451118
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 0
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": -3.248277111101165,
       "cc_by_type": {
        "choice": 68.42105263157895,
        "noul": -100,
        "score": 21.834116035117557
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.19047619047619047,
       "cc": -28.752891240849554,
       "cc_by_type": {
        "choice": 9.547738693467341,
        "noul": -100,
        "score": 4.193587583984004
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.15384615384615385,
       "cc": -37.98670463078509,
       "cc_by_type": {
        "choice": 5.660377358490565,
        "noul": -100,
        "score": -19.620491250845838
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.15625,
       "cc": -32.51983448550218,
       "cc_by_type": {
        "choice": -3.585657370517929,
        "noul": -100,
        "score": 6.0261539140113785
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.09090909090909091,
       "cc": -35.388151349291284,
       "cc_by_type": {
        "choice": -14.558472553699291,
        "noul": -100,
        "score": 8.394018505825429
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      61.644678187031296,
      70.74968953890287
     ],
     "I": [
      0.8350318363608307,
      4.617476925221922
     ],
     "capability": [
      31.94442581392808,
      37.14219934722527
     ],
     "composite": {
      "A": [
       0.0008967609092971513,
       0.12984410368517996
      ],
      "B": [
       0.0005711694021380616,
       0.08896822577663006
      ],
      "C": [
       0.000622750631456355,
       0.09016951644804166
      ]
     },
     "seed": 26
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 66.68251917899094,
     "capability": 34.64663304369169,
     "intelligence": 2.6107469083924393
    },
    "per_type": {
     "choice": {
      "accuracy": 0.268,
      "calibration": 67.60445581391808,
      "cc": 5.164968042839871,
      "cc_by_type": {
       "choice": 5.164968042839871
      },
      "cc_tiered": 7.832240725177318,
      "cc_tiered_open": 11.480997011004092,
      "cc_tiered_sealed": 4.183484439350544,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 5.164968042839871
     },
     "noul": {
      "accuracy": 0,
      "calibration": 59.27225925455259,
      "cc": -100,
      "cc_by_type": {
       "noul": -100
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2613333333333333,
      "calibration": 73.17084246850217,
      "cc": -4.528144639423148,
      "cc_by_type": {
       "score": -4.528144639423148
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 59.34147162065773,
     "capability": 31.140405932959595,
     "intelligence": 2.939340245261466
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.01973059249456439,
    "Emphasis on Accuracy 60:20:20": 0.011546588753666046,
    "Emphasis on Cost 20:20:60": 0.030774419703851856,
    "Emphasis on Speed 20:60:20": 0.030350892509696595,
    "Intelligence only": 0.0071179397650143975
   }
  },
  {
   "adapter_id": "images/mxbai-rerank-base-v2/spec.json:reranker_adapter.RerankerAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mixedbread",
   "axes": {
    "calibration": 86.60772459153094,
    "cost": 60.33589314238483,
    "intelligence": 1.140707327109041,
    "speed": 92.05600895369167
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06855480504522303,
      "ece_score": 86.28903899095539,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 86.28903899095539,
      "score_before_time_limit": 86.28903899095539,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 1,
      "brier": 0.25057053855990563,
      "ece": 0.07454411727378144,
      "n": 375,
      "score": 85.09117654524371,
      "score_before_time_limit": 85.09117654524371,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 1,
      "interval90_width": 1,
      "n": 375,
      "nrps": 0.17161399632774915,
      "score": 88.44295823839374,
      "score_before_time_limit": 88.44295823839374,
      "time_limit_failed_share": 0,
      "top_ece": 0.029763419452187988
     }
    },
    "score": 86.60772459153094
   },
   "capability": 43.87421595931999,
   "class": "reranker",
   "composite_ci95": {
    "A": [
     4.213790740997237e-06,
     0.03217925832474849
    ],
    "B": [
     2.6407563817968135e-06,
     0.021162373242485297
    ],
    "C": [
     2.9262435701369706e-06,
     0.02234670716996423
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: size-class proxy (deepinfra:Qwen/Qwen3-Reranker-0.6B); no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020996016009852216
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Mixedbread mxbai-rerank-base-v2",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 1.5821442040322362,
    "I_sealed": 0.6992704501858459,
    "base": 1.140707327109041,
    "excess": -1.6905138885918536,
    "gap": 0.8828737538463903,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 4.746432612096709,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -5.081081081081076,
       "hard": -0.03145643284050508,
       "judge": 1.1206328279499007,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -11.030065754992213,
       "hard": -1.5426620524559675,
       "judge": 8.497895555670832,
       "standard": -21.144844232023672
      }
     },
     "sealed|choice": {
      "cc": 0,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -7.75256817677922,
       "hard": -4.355203619909522,
       "judge": -1.4897579143388995,
       "standard": 0.23401896785319723
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "sealed|score": {
      "cc": 2.0978113505575378,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -4.830772427457819,
       "hard": 1.5240605072453395,
       "judge": 6.561942168694756,
       "standard": 0.013408698983785605
      }
     }
    }
   },
   "jevbench_score": 0.0022737744722840765,
   "key": "mxbai-rerank-base-v2",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive_rate": 0,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.60772459153094,
     "cost": 60.33589314238483,
     "intelligence": 1.140707327109041,
     "speed": 92.58758170313658
    },
    "board_release": "v1.6.43",
    "capability": 43.87421595931999,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 104,
    "capability_rank": 132,
    "composite_ci95": {
     "A": [
      4.213826902762912e-06,
      0.0321843481372824
     ],
     "B": [
      2.6407677436324784e-06,
      0.02116413476940045
     ],
     "C": [
      2.9262686824742456e-06,
      0.022350241762001666
     ]
    },
    "jevbench_score": 0.002273929358844764,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 190,
    "rank": 234,
    "ranks": {
     "A": 234,
     "B": 234,
     "C": 234,
     "capability": 193
    },
    "scores": {
     "A": 0.002273929358844764,
     "B": 0.0014520695133694007,
     "C": 0.001579117610308864
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21285173026844859,
     "p50_s_raw": 0.031425865134224296,
     "p95_s_adjusted": 0.2589194315019995,
     "p95_s_raw": 0.05445971575099974
    },
    "views": {
     "Balanced 33:33:33": 0.0017272292257436567,
     "Emphasis on Accuracy 60:20:20": 0.0009793418989392745,
     "Emphasis on Cost 20:20:60": 0.002776895162759242,
     "Emphasis on Speed 20:60:20": 0.002811535213497941,
     "Intelligence only": 0.0005937213753409678
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 275,
   "ranked": true,
   "ranks": {
    "A": 275,
    "B": 275,
    "C": 275,
    "capability": 232
   },
   "repo": "https://huggingface.co/mixedbread-ai/mxbai-rerank-base-v2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.0022737744722840765,
    "B": 0.0014520189845594442,
    "C": 0.0015790100501972756
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21420394033193588,
    "p50_s_raw": 0.03210197016596794,
    "p95_s_adjusted": 0.2907847651280462,
    "p95_s_raw": 0.0703923825640231
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "494M cross-encoder",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.5709999496809,
      89.28164941290872
     ],
     "I": [
      0.13834674179555617,
      2.8157994354473197
     ],
     "capability": [
      41.27690031534251,
      45.59940972334693
     ],
     "composite": {
      "A": [
       4.213790740997237e-06,
       0.03217925832474849
      ],
      "B": [
       2.6407563817968135e-06,
       0.021162373242485297
      ],
      "C": [
       2.9262435701369706e-06,
       0.02234670716996423
      ]
     },
     "seed": 69
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.60772459153094,
     "capability": 43.87421595931999,
     "intelligence": 1.140707327109041
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2,
      "calibration": 86.28903899095539,
      "cc": -2.3085740268844592,
      "cc_by_type": {
       "choice": -2.3085740268844592
      },
      "cc_tiered": 2.3732163060483544,
      "cc_tiered_open": 4.746432612096709,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 0
     },
     "noul": {
      "accuracy": 0,
      "calibration": 85.09117654524371,
      "cc": -100,
      "cc_by_type": {
       "noul": -100
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.3413333333333333,
      "calibration": 88.44295823839374,
      "cc": 0.3851131600845714,
      "cc_by_type": {
       "score": 0.3851131600845714
      },
      "cc_tiered": 1.0489056752787689,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 2.0978113505575378,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.3851131600845714
     }
    },
    "raw_on_A": {
     "calibration": 84.06228474550946,
     "capability": 42.68728856938789,
     "intelligence": 1.312292393266321
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.0017271100746280466,
    "Emphasis on Accuracy 60:20:20": 0.0009793189142918838,
    "Emphasis on Cost 20:20:60": 0.0027767103766015502,
    "Emphasis on Speed 20:60:20": 0.002810967015237966,
    "Intelligence only": 0.0005937213753409678
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "nagisanzenin",
   "axes": {
    "calibration": 85.14254261258804,
    "cost": 57.35797715995501,
    "intelligence": 62.38818589248167,
    "speed": 85.82001330475426
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07628781316710302,
      "ece_score": 84.7424373665794,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 84.7424373665794,
      "score_before_time_limit": 84.7424373665794,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.1554959785522788,
      "brier": 0.10066812652875887,
      "ece": 0.08008409057957327,
      "n": 373,
      "score": 83.98318188408534,
      "score_before_time_limit": 83.98318188408534,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9327956989247311,
      "interval90_width": 0.258736559139785,
      "n": 372,
      "nrps": 0.07096566496914646,
      "score": 86.70200858709943,
      "score_before_time_limit": 86.70200858709943,
      "time_limit_failed_share": 0,
      "top_ece": 0.09749708164443252
     }
    },
    "score": 85.14254261258804
   },
   "capability": 73.76536425253485,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 762,436 input + 0 output tokens over 1,485 decisions = 513.424915825 tokens/decision. GPU-time cost USD 0.0263876528116565/1,000; old list-price cost USD 0.215638464646465/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0263876528116565/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.21348208
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02638765281165652
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Nagi-ENORMOUS 27B (nagisanzenin, Qwen3.8-27B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's SDK loaded in-process (nagi.load_enormous) behind the evaluator's wrapper.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 67.77912378162935,
    "I_sealed": 56.997248003334,
    "base": 62.38818589248167,
    "excess": 0.9423490079396402,
    "gap": 10.781875778295351,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 74.75336686147108,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 75.46398238439761,
       "judge": 76.26895187870798,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 65.67632850241546,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 50,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 62.90767598100152,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.48231371851977,
       "hard": 64.9557142549451,
       "judge": 69.52499554842026,
       "standard": 48.59830121322711
      }
     },
     "sealed|choice": {
      "cc": 73.1916523589463,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 86.6581956797967,
       "hard": 68.77529779114143,
       "judge": 74.43682664054849,
       "standard": 73.4233284117275
      }
     },
     "sealed|noul": {
      "cc": 47.06459263844751,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 68.96551724137932,
       "hard": 45.45454545454546,
       "judge": 38.46153846153846,
       "standard": 52.23880597014925
      }
     },
     "sealed|score": {
      "cc": 50.7354990126082,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 68.6310910781143,
       "hard": 46.3612306455534,
       "judge": 49.43194058927436,
       "standard": 52.49157734896552
      }
     }
    }
   },
   "jevbench_score": 70.34779585768042,
   "key": "nagi-enormous",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nagisanzeninz/Nagi-ENORMOUS@0d8e052d698461ecd49c4f39a09f6bd0179a1d5c",
   "noul_decisive": {
    "acc_among_decisive": 0.9142857142857143,
    "decisive": 315,
    "decisive_rate": 0.8445040214477212,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.14254261258804,
     "cost": 29.98821307073638,
     "intelligence": 62.38818589248167,
     "speed": 86.16105911802293
    },
    "board_release": "v1.6.43",
    "capability": 73.76536425253485,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      19.294091538220005,
      20.0652017609536
     ],
     "B": [
      19.539777428272505,
      20.734381755643742
     ],
     "C": [
      17.53840599334894,
      20.0652017609536
     ]
    },
    "jevbench_score": 19.784711166742767,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 97,
    "rank": 132,
    "ranks": {
     "A": 132,
     "B": 126,
     "C": 116,
     "capability": 50
    },
    "scores": {
     "A": 19.784711166742767,
     "B": 20.264621768982312,
     "C": 19.784711166742767
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4519195217639208,
     "p50_s_raw": 0.1509597608819604,
     "p95_s_adjusted": 0.5355906491167843,
     "p95_s_raw": 0.19279532455839216
    },
    "views": {
     "Balanced 33:33:33": 17.696438535382505,
     "Emphasis on Accuracy 60:20:20": 19.331602164737944,
     "Emphasis on Cost 20:20:60": 14.087313813773893,
     "Emphasis on Speed 20:60:20": 21.36249462717993,
     "Intelligence only": 22.442101625156123
    }
   },
   "rank": 73,
   "ranked": true,
   "ranks": {
    "A": 73,
    "B": 65,
    "C": 57,
    "capability": 68
   },
   "repo": "https://huggingface.co/nagisanzeninz/Nagi-ENORMOUS",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.34779585768042,
    "B": 68.59743614293767,
    "C": 70.34779585768042
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4575406416319311,
    "p50_s_raw": 0.15377032081596553,
    "p95_s_adjusted": 0.5722278523724526,
    "p95_s_raw": 0.2111139261862263
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 74.14032183781576,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 47.4209655134473
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 74.14032183781576
      },
      "calibration_gen": {
       "accuracy": 0.6732673267326733,
       "cc": 54.21132684915775,
       "cc_by_type": {
        "choice": 68.18663838812301,
        "noul": 43.99999999999999,
        "score": 50.44734215935023
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 54.21132684915775
      },
      "composite_gen": {
       "accuracy": 0.8514851485148515,
       "cc": 79.6200178020504,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 52.77152585420901
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 79.6200178020504
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 82.42374252124812,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 87.27122756374432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 82.42374252124812
      },
      "jaggedness": {
       "accuracy": 0.5666666666666667,
       "cc": 25.273315341162046,
       "cc_by_type": {
        "choice": 45.13955726660251,
        "noul": -18.918918918918916,
        "score": 49.59930767580255
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 25.273315341162046
      },
      "llm_router": {
       "accuracy": 0.9105431309904153,
       "cc": 82.02065127885221,
       "cc_by_type": {
        "choice": 91.28290921898389,
        "noul": 95.8762886597938,
        "score": 58.90275595777894
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 82.02065127885221
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4444444444444444,
       "cc": 6.669224859541649,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -25,
        "score": 19.23447870233629
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 6.669224859541649
      },
      "multilingual": {
       "accuracy": 0.7345454545454545,
       "cc": 57.00736526474546,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": 38.888888888888886,
        "score": 60.499688809488326
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 57.00736526474546
      },
      "negation_pair_gen": {
       "accuracy": 0.8488372093023255,
       "cc": 61.959669648822455,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 46.66666666666666,
        "score": 45.87900894646737
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 61.959669648822455
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 73.20537560651158,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 80,
        "score": 57.63414483755276
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 73.20537560651158
      },
      "usecase": {
       "accuracy": 0.7045454545454546,
       "cc": 54.04431686728915,
       "cc_by_type": {
        "choice": 68.1372549019608,
        "noul": 41.66666666666667,
        "score": 52.32902903323995
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 54.04431686728915
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 22.777334487427282,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": -24.525139394861007
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 22.777334487427282
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 23.3844609455102,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 3.4867161698639237
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 23.3844609455102
      },
      "de": {
       "accuracy": 0.875,
       "cc": 83.78205425673583,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 75.239968079942
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.78205425673583
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 38.730441914975806,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 99.80398868719931
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 38.730441914975806
      },
      "en": {
       "accuracy": 0.7688966116420504,
       "cc": 62.08334185157287,
       "cc_by_type": {
        "choice": 74.95759143774178,
        "noul": 56.83453237410072,
        "score": 54.457901742876125
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 62.08334185157287
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 66.46236509889698,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 9.17502351855064
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.46236509889698
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.48213085184086,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 98.89994446809092
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.48213085184086
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 82.9428569359917,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 91.18151198444569
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 82.9428569359917
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 49.72408656861362,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 25,
        "score": 57.691812778466556
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 49.72408656861362
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 58.09938627602248,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 2.357394993305184
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 58.09938627602248
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 21.74234134927224,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 0,
        "score": -6.178853632882242
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 21.74234134927224
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -1.3052903918953547,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 40.528573268758386
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.0491525522278,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.1474576566834
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.0491525522278
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -20.38367168727892,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": -9.202963113784811
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 76.55637870233811,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80,
        "score": 64.9040070338023
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 76.55637870233811
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 74.98400673194925,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 67.37983398488723
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 74.98400673194925
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 59.284793741948214,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 68.91821101307869
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 59.284793741948214
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 68.73685288229498,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100,
        "score": 49.68881951645017
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 68.73685288229498
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 86.03268404130108,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 90.35611664003227
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 86.03268404130108
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 54.74146896611071,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 30.891073564998806
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 54.74146896611071
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8,
       "cc": 74.1671291069481,
       "cc_by_type": {
        "choice": 74.1671291069481
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 74.1671291069481
      },
      "noul": {
       "accuracy": 0.768,
       "cc": 53.6,
       "cc_by_type": {
        "noul": 53.6
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 53.6
      },
      "score": {
       "accuracy": 0.6666666666666666,
       "cc": 54.012205085152345,
       "cc_by_type": {
        "score": 54.012205085152345
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 54.012205085152345
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -5.814728435567205,
       "cc_by_type": {
        "choice": 20,
        "noul": -25,
        "score": -12.444185306701616
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 65.96125238263933,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 16.517297520588812
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 65.96125238263933
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 40.62716551888533,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100,
        "score": -61.858340841718
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 40.62716551888533
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 2.0832411267705617,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 33.33333333333333,
        "score": -33.51635849103334
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 2.0832411267705617
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 39.13973735127555,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 65.24529901034839
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 39.13973735127555
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -25.614597750300508,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 33.33333333333333,
        "score": -159.86266117542982
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.7333333333333333,
       "cc": 57.204794917237045,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 42.85714285714286,
        "score": 58.16900660045062
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 57.204794917237045
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 29.714186153960508,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": -50,
        "score": 60.048481806829244
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 29.714186153960508
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 0.8791936057861172,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 25.13758081735835
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0.8791936057861172
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 55.57642841921321,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 66.66666666666667,
        "score": 51.45448154600507
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 55.57642841921321
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 51.56691874020171,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 33.33333333333333,
        "score": 44.62323684076017
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 51.56691874020171
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": 11.911912271535977,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 31.434661545790732
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 11.911912271535977
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 72.50539113615088,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 69.03248818772326
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 72.50539113615088
      },
      "model_routing": {
       "accuracy": 0.8940092165898618,
       "cc": 78.06075539889343,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 93.93939393939394,
        "score": 46.087028101442215
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 78.06075539889343
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 17.71194333648423,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 64.65214478872332
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 17.71194333648423
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 35.85502771771261,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 74.32460033048231
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 35.85502771771261
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 30.833875246891477,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": 22.058275986979837
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 30.833875246891477
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 21.590370642594802,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 56.07545975387135
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 21.590370642594802
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.11485034921368,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.34455104764109
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.11485034921368
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.73695118367363,
      87.37702121824532
     ],
     "I": [
      57.11710491178149,
      66.34463835228868
     ],
     "capability": [
      69.6246461730435,
      76.14865327125807
     ],
     "composite": null,
     "seed": 50
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.14254261258804,
     "capability": 73.76536425253485,
     "intelligence": 62.38818589248167
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8,
      "calibration": 84.7424373665794,
      "cc": 74.1671291069481,
      "cc_by_type": {
       "choice": 74.1671291069481
      },
      "cc_tiered": 73.97250961020869,
      "cc_tiered_open": 74.75336686147108,
      "cc_tiered_sealed": 73.1916523589463,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 74.1671291069481
     },
     "noul": {
      "accuracy": 0.768,
      "calibration": 83.98318188408534,
      "cc": 53.6,
      "cc_by_type": {
       "noul": 53.6
      },
      "cc_tiered": 56.37046057043148,
      "cc_tiered_open": 65.67632850241546,
      "cc_tiered_sealed": 47.06459263844751,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 53.6
     },
     "score": {
      "accuracy": 0.6666666666666666,
      "calibration": 86.70200858709943,
      "cc": 54.012205085152345,
      "cc_by_type": {
       "score": 54.012205085152345
      },
      "cc_tiered": 56.82158749680486,
      "cc_tiered_open": 62.90767598100152,
      "cc_tiered_sealed": 50.7354990126082,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.012205085152345
     }
    },
    "raw_on_A": {
     "calibration": 84.73824539717386,
     "capability": 73.41666063559657,
     "intelligence": 62.09507587401927
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.4962353566332,
    "Emphasis on Accuracy 60:20:20": 64.78976134668866,
    "Emphasis on Cost 20:20:60": 62.512448460223034,
    "Emphasis on Speed 20:60:20": 73.0781291923389,
    "Intelligence only": 62.38818589248167
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "nagisanzenin",
   "axes": {
    "calibration": 73.2732669877037,
    "cost": null,
    "intelligence": 54.80664491424396,
    "speed": 88.38293814019335
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14730316908633123,
      "ece_score": 70.53936618273376,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 70.53936618273376,
      "score_before_time_limit": 70.53936618273376,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.1420911528150134,
      "brier": 0.13578925025367564,
      "ece": 0.11265263438596824,
      "n": 373,
      "score": 77.46947312280635,
      "score_before_time_limit": 77.46947312280635,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8521505376344086,
      "interval90_width": 0.2540322580645162,
      "n": 372,
      "nrps": 0.1041654308585557,
      "score": 71.81096165757103,
      "score_before_time_limit": 71.81096165757103,
      "time_limit_failed_share": 0,
      "top_ece": 0.22980766799501195
     }
    },
    "score": 73.2732669877037
   },
   "capability": 64.03995595097383,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 3948e3f1..., and the later re-freezes, which only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base google/gemma-4-12B-it (the module lists it in UNLISTED_BASE_MODELS, so reference() returns no price); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens as reported by the evaluator's wrapper (attention-mask ones over every forward pass: input 764,280, output_tokens 0).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Nagi-HUGE 12B (nagisanzenin, Gemma-4-12B LoRA)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's SDK loaded in-process (nagi.load_huge) behind the evaluator's wrapper.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 59.531691789779515,
    "I_sealed": 50.081598038708414,
    "base": 54.80664491424396,
    "excess": -0.3894330192846098,
    "gap": 9.450093751071101,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 71.98506861847196,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 68.35860250494397,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 52.719806763285035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 30.000000000000004,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 53.89019998758157,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 59.56725028103613,
       "hard": 59.87843833211595,
       "judge": 57.63522131418832,
       "standard": 33.4576661618754
      }
     },
     "sealed|choice": {
      "cc": 68.46211926245671,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 84.75222363405337,
       "hard": 62.530357349369716,
       "judge": 68.26640548481882,
       "standard": 72.47416156928918
      }
     },
     "sealed|noul": {
      "cc": 44.28703360458379,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 41.81818181818182,
       "judge": 29.230769230769237,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 37.49564124908475,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 56.18011440890034,
       "hard": 32.5186886432668,
       "judge": 31.435392540978093,
       "standard": 47.19768294297284
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "nagi-huge",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "listed",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nagisanzeninz/Nagi-HUGE@3e0a58b19116afca7abf7f854d82ba85f5de5c4b",
   "not_ranked_because": "unpriced: google/gemma-4-12B-it is in the frozen module's UNLISTED_BASE_MODELS (no eligible base-model price); one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.878125,
    "decisive": 320,
    "decisive_rate": 0.8579088471849866,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.2732669877037,
     "cost": null,
     "intelligence": 54.80664491424396,
     "speed": 88.62940634950104
    },
    "board_release": "v1.6.43",
    "capability": 64.03995595097383,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.34666025787591936,
     "p50_s_raw": 0.09833012893795967,
     "p95_s_adjusted": 0.3955080335494131,
     "p95_s_raw": 0.12275401677470654
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/nagisanzeninz/Nagi-HUGE",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.35058780908584597,
    "p50_s_raw": 0.10029390454292297,
    "p95_s_adjusted": 0.4139132943470031,
    "p95_s_raw": 0.13195664717350153
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 80.8965045441356,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 59.35618029907346
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 80.8965045441356
      },
      "calibration_gen": {
       "accuracy": 0.5742574257425742,
       "cc": 41.022478364333296,
       "cc_by_type": {
        "choice": 42.73594909862143,
        "noul": 43.99999999999999,
        "score": 36.331485994378454
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 41.022478364333296
      },
      "composite_gen": {
       "accuracy": 0.7227722772277227,
       "cc": 54.3475547262935,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": 40.74074074074075,
        "score": 48.318183600741364
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 54.3475547262935
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 88.61842154243884,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 65.8552646273165
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 88.61842154243884
      },
      "jaggedness": {
       "accuracy": 0.5666666666666667,
       "cc": 27.382018300134657,
       "cc_by_type": {
        "choice": 52.358036573628496,
        "noul": 2.7027027027026973,
        "score": 27.08531562407278
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 27.382018300134657
      },
      "llm_router": {
       "accuracy": 0.8466453674121406,
       "cc": 74.9947462424799,
       "cc_by_type": {
        "choice": 81.77335563969358,
        "noul": 83.50515463917525,
        "score": 59.70572844857087
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 74.9947462424799
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 20.044670000518725,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 0,
        "score": -15.123721957206726
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 20.044670000518725
      },
      "multilingual": {
       "accuracy": 0.730909090909091,
       "cc": 57.472837311308034,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": 52.77777777777777,
        "score": 49.96352515712449
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 57.472837311308034
      },
      "negation_pair_gen": {
       "accuracy": 0.6744186046511628,
       "cc": 31.85947400478956,
       "cc_by_type": {
        "choice": 76.66666666666666,
        "noul": -6.666666666666665,
        "score": 25.578422014368694
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 31.85947400478956
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 81.2963361578178,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100,
        "score": 70.91603550048045
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 81.2963361578178
      },
      "usecase": {
       "accuracy": 0.6363636363636364,
       "cc": 48.42747769597158,
       "cc_by_type": {
        "choice": 63.23529411764707,
        "noul": 41.66666666666667,
        "score": 40.380472303601
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 48.42747769597158
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 7.208501158443632,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": -90.27925842943101
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 7.208501158443632
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 52.975324368289364,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 46.32996696047484
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 52.975324368289364
      },
      "de": {
       "accuracy": 0.875,
       "cc": 84.10830849392268,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100,
        "score": 84.18333256141403
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 84.10830849392268
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.5874788778112,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 99.94865637272596
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.5874788778112
      },
      "en": {
       "accuracy": 0.7072111207645526,
       "cc": 52.781473595451,
       "cc_by_type": {
        "choice": 69.0264946729964,
        "noul": 47.482014388489205,
        "score": 41.835911724867394
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 52.781473595451
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.1400106830659,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 29.2079602710574
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.1400106830659
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.7975629613487,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 99.84624079661444
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.7975629613487
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 84.29596567394786,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 74.06436761007889
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.29596567394786
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 41.946148025454015,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 25,
        "score": 0.8384440763620549
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 41.946148025454015
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 71.63085300784105,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 28.922177106142033
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.63085300784105
      },
      "ja": {
       "accuracy": 0.44,
       "cc": 12.01915193855757,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 0,
        "score": 7.542761613925297
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 12.01915193855757
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 64.63002166245748,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100,
        "score": 38.33450943181689
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 64.63002166245748
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.16468569071083,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.4940570721325
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.16468569071083
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 29.144393403202685,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 6.047898824326681
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 29.144393403202685
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 56.66691195109527,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 19.999999999999996,
        "score": 57.61817131667982
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 56.66691195109527
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.41158446563735,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 53.6625671859515
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.41158446563735
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 44.08012594525695,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 23.304207623004913
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 44.08012594525695
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 38.30087713735259,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50,
        "score": -13.358238153159618
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 38.30087713735259
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 81.29992212215188,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 76.15783088258468
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 81.29992212215188
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 37.21820113197382,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": -21.678729937411866
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 37.21820113197382
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7653333333333333,
       "cc": 69.68943148548576,
       "cc_by_type": {
        "choice": 69.68943148548576
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 69.68943148548576
      },
      "noul": {
       "accuracy": 0.7493333333333333,
       "cc": 49.86666666666666,
       "cc_by_type": {
        "noul": 49.86666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 49.86666666666666
      },
      "score": {
       "accuracy": 0.584,
       "cc": 42.51279672323708,
       "cc_by_type": {
        "score": 42.51279672323708
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 42.51279672323708
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 18.68109795677604,
       "cc_by_type": {
        "choice": 52,
        "noul": 0,
        "score": 4.043293870328113
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 18.68109795677604
      },
      "code_linting": {
       "accuracy": 0.6666666666666666,
       "cc": 45.116373885865976,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 77.77777777777777,
        "score": -5.161575374838234
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 45.116373885865976
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 50.81996606870461,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 34.98022340936585
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 50.81996606870461
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -18.91273698918377,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": -29.83762617222967
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 50.965624322682,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 48.54904688108945
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 50.965624322682
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -45.01234945271016,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 0,
        "score": -184.72258294932544
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 34.253607619886445,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 14.28571428571428,
        "score": 27.690794848454857
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 34.253607619886445
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": 4.190740847971121,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": -100,
        "score": 54.38406923380883
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 4.190740847971121
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 50.32081411651301,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 53.46244234953903
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 50.32081411651301
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 68.33470459909553,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100,
        "score": 43.548011013560675
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 68.33470459909553
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 49.021948938186966,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 3.1381982357495453
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 49.021948938186966
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 31.490484817207232,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 25.65425015054643
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 31.490484817207232
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 68.93444315573477,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 60.00000000000001,
        "score": 46.80332946720428
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.93444315573477
      },
      "model_routing": {
       "accuracy": 0.8663594470046083,
       "cc": 78.12484590988917,
       "cc_by_type": {
        "choice": 87.14285714285714,
        "noul": 100,
        "score": 47.2316805868104
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 78.12484590988917
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 39.09003196881363,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 40.302725464982146
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 39.09003196881363
      },
      "recruiting": {
       "accuracy": 0.8888888888888888,
       "cc": 87.47734221450266,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100,
        "score": 79.14511856551354
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 87.47734221450266
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 33.94641567016526,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": 16.617572133648473
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 33.94641567016526
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 28.08332853716604,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": 33.33333333333333,
        "score": 37.87317401729523
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 28.08332853716604
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 95.87265733076127,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 98.2941285759137
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 95.87265733076127
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.82121675320265,
      76.3898801463225
     ],
     "I": [
      49.533417058444485,
      59.0670299888095
     ],
     "capability": [
      59.842759105481505,
      67.1284484360694
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 51
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.2732669877037,
     "capability": 64.03995595097383,
     "intelligence": 54.80664491424396
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7653333333333333,
      "calibration": 70.53936618273376,
      "cc": 69.68943148548576,
      "cc_by_type": {
       "choice": 69.68943148548576
      },
      "cc_tiered": 70.22359394046433,
      "cc_tiered_open": 71.98506861847196,
      "cc_tiered_sealed": 68.46211926245671,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.68943148548576
     },
     "noul": {
      "accuracy": 0.7493333333333333,
      "calibration": 77.46947312280635,
      "cc": 49.86666666666666,
      "cc_by_type": {
       "noul": 49.86666666666666
      },
      "cc_tiered": 48.50342018393441,
      "cc_tiered_open": 52.719806763285035,
      "cc_tiered_sealed": 44.28703360458379,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 49.86666666666666
     },
     "score": {
      "accuracy": 0.584,
      "calibration": 71.81096165757103,
      "cc": 42.51279672323708,
      "cc_by_type": {
       "score": 42.51279672323708
      },
      "cc_tiered": 45.69292061833316,
      "cc_tiered_open": 53.89019998758157,
      "cc_tiered_sealed": 37.49564124908475,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 42.51279672323708
     }
    },
    "raw_on_A": {
     "calibration": 73.92409836881791,
     "capability": 64.51989601789064,
     "intelligence": 55.11569366696338
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "pst2154",
   "axes": {
    "calibration": 89.03920345557668,
    "cost": null,
    "intelligence": 49.60038985253669,
    "speed": 87.35910345792061
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.031840143420803706,
      "ece_score": 93.63197131583925,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 93.63197131583925,
      "score_before_time_limit": 93.63197131583925,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.30400000000000005,
      "brier": 0.10770403214312142,
      "ece": 0.06945449416016128,
      "n": 375,
      "score": 86.10910116796775,
      "score_before_time_limit": 86.10910116796775,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.928,
      "interval90_width": 0.37866666666666676,
      "n": 375,
      "nrps": 0.09148110518890483,
      "score": 87.37653788292307,
      "score_before_time_limit": 87.37653788292307,
      "time_limit_failed_share": 0,
      "top_ece": 0.0804940685763168
     }
    },
    "score": 89.03920345557668
   },
   "capability": 69.31979665405669,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module RAISES on the declared base nvidia/Nemotron-Labs-Diffusion-8B ('base model lacks an explicit market-listing decision') and the dated snapshot carries no listing of that family, so this row carries no pricing_base and is listed, measured and shown WITHOUT a cost number and WITHOUT a rank - the pre-declared default (Sage 1.4, Celeris Option A). No rate is invented. The predecessor nemotron-diffusion-8b row's usd_per_1000_override is a v1.5.4 historical carry on a different draw and is deliberately NOT reused. Pricing remains unresolved; the server reports both token fields, so a later pricing decision needs no re-measurement.",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Nemotron Deciffusion 8B v2 (pst2154)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-15",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (United States; measurement receipts on file), 1x NVIDIA RTX A6000 48 GB - different hardware from the L40S and GB200 the author quotes and from the H100/RTX 5090 rows of this draw. Started from the author's OWN published container, by DIGEST ghcr.io/pst2154/nemotron-deciffusion@sha256:944c0af4967ebd1e461a3b1faa9d16db197288af24393b21c917617461f35276 (image config sha256:0a4bd2d039b5c2da9ff8b181232ee3714f6b38e31bd9a3100a29326068d6f802), in the author's own documented sandboxed form: docker run --gpus all --shm-size=8g --network none --read-only --tmpfs /tmp:rw,exec,size=16g --user 1000:1000 --cap-drop ALL --security-opt no-new-privileges -v <checkpoint>:/models/checkpoint:ro, with QUANTIZATION=none (BF16) and MAX_MODEL_LEN=16384 and every other default of the image (ATTENTION=noncausal, COMPACT_OVER=8192, COMPACT_BUDGET=8192, MAX_BATCHED_TOKENS=65536, --no-enable-prefix-caching, --max-logprobs 1052, --logprobs-mode processed_logits, vLLM nightly fe4df3df0eb7 with the author's overlay). /health reported back quantization none, attention noncausal, max_prompt_tokens 16382. All ten checkpoint files verified against the author's own SHA256SUMS (sha256sum -c, 10/10 OK) BEFORE the container started. One request at a time from the unchanged runner, which reached the server through the container's own network namespace with nsenter - the container published no port and had no network at all. The cut was proven before the sealed input was staged (only lo, no routes, a name probe and a bare-IP probe both failed) and re-proven immediately before the first sealed request, with a synthetic smoke gate passed in between (under the pre-registration). 1,500/1,500 answered, HTTP 200 on every item, raw p50 105 ms / p95 180 ms / max 1.82 s.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 55.54041453254173,
    "I_sealed": 43.660365172531655,
    "base": 49.60038985253669,
    "excess": 2.0405225896543655,
    "gap": 11.880049360010076,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.4454905515748,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 84.90091223655237,
       "judge": 72.31377719182598,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 44.98550724637681,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 39.13043478260869,
       "judge": 19.999999999999996,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 46.19024579967359,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.27224719217729,
       "hard": 41.00474552808914,
       "judge": 52.73987117942826,
       "standard": 34.19580757695866
      }
     },
     "sealed|choice": {
      "cc": 66.84585672814276,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 65.65282757025557,
       "judge": 56.80705190989228,
       "standard": 71.52499472685088
      }
     },
     "sealed|noul": {
      "cc": 23.96819855389078,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 44.827586206896555,
       "hard": 27.27272727272727,
       "judge": 7.692307692307687,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 40.167040235561444,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 62.908566475040885,
       "hard": 39.62773652825137,
       "judge": 34.70949534264651,
       "standard": 38.061201869814276
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "nemotron-deciffusion-8b-v2",
   "last_measured_on": "2026-10-15",
   "licence": "NVIDIA Nemotron Open Model License (https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-nemotron-open-model-license/); base nvidia/Nemotron-Labs-Diffusion-8B",
   "listing": "listed",
   "measured_in": "v1.6.22",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nemotron-deciffusion/Nemotron-Deciffusion-8B-v2@1dea38015e86e101eb978a33fa493818fc71e366",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base nvidia/Nemotron-Labs-Diffusion-8B and raises rather than estimating; a future published pricing update may add an entry",
   "noul_decisive": {
    "acc_among_decisive": 0.9501915708812261,
    "decisive": 261,
    "decisive_rate": 0.696,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.03920345557668,
     "cost": null,
     "intelligence": 49.60038985253669,
     "speed": 87.70433528726323
    },
    "board_release": "v1.6.43",
    "capability": 69.31979665405669,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.22",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.35293248631060126,
     "p50_s_raw": 0.10146624315530062,
     "p95_s_adjusted": 0.48070078976452346,
     "p95_s_raw": 0.16535039488226172
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/nemotron-deciffusion/Nemotron-Deciffusion-8B-v2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3605954047292471,
    "p50_s_raw": 0.10529770236462355,
    "p95_s_adjusted": 0.5094123460352421,
    "p95_s_raw": 0.17970617301762104
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 69.81508925384576,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 17.778601094870574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 69.81508925384576
      },
      "calibration_gen": {
       "accuracy": 0.5247524752475248,
       "cc": 24.130548030105405,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": -19.999999999999996,
        "score": 36.930350346943996
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 24.130548030105405
      },
      "composite_gen": {
       "accuracy": 0.8118811881188119,
       "cc": 62.586444350419164,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 40.74074074074075,
        "score": 56.77468987149234
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 62.586444350419164
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 76.48515065368318,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 69.45545196104952
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 76.48515065368318
      },
      "jaggedness": {
       "accuracy": 0.5611111111111111,
       "cc": 22.292630536155855,
       "cc_by_type": {
        "choice": 63.90760346487007,
        "noul": -18.918918918918916,
        "score": 21.889207062516412
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 22.292630536155855
      },
      "llm_router": {
       "accuracy": 0.7635782747603834,
       "cc": 64.81045782636264,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 79.38144329896907,
        "score": 49.91829330418326
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 64.81045782636264
      },
      "long_state": {
       "accuracy": 0.5333333333333333,
       "cc": -14.49955948045043,
       "cc_by_type": {
        "choice": 48.387096774193544,
        "noul": -100,
        "score": 8.114224784455171
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -14.464526914190303,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -25,
        "score": -31.795642598241013
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6727272727272727,
       "cc": 43.55075461566889,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": 16.666666666666675,
        "score": 48.22100637499275
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 43.55075461566889
      },
      "negation_pair_gen": {
       "accuracy": 0.7209302325581395,
       "cc": 40.54640345029721,
       "cc_by_type": {
        "choice": 86.66666666666667,
        "noul": 6.666666666666665,
        "score": 28.305877017558302
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 40.54640345029721
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 76.80928525816542,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100,
        "score": 57.45488280152327
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 76.80928525816542
      },
      "usecase": {
       "accuracy": 0.6772727272727272,
       "cc": 46.174276523358934,
       "cc_by_type": {
        "choice": 70.58823529411767,
        "noul": 25,
        "score": 42.934594275959135
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 46.174276523358934
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 7.324936282948684,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 100,
        "score": -101.83471496067776
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 7.324936282948684
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 43.555049861840054,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 18.06914344112691
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 43.555049861840054
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 60.20540912770944,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75,
        "score": 37.474634462774326
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 60.20540912770944
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 32.087424203928286,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 96.44849235107722
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 32.087424203928286
      },
      "en": {
       "accuracy": 0.7046046915725456,
       "cc": 49.66247129995006,
       "cc_by_type": {
        "choice": 72.10187818064216,
        "noul": 33.81294964028776,
        "score": 43.07258607892028
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 49.66247129995006
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 67.89447005427888,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75,
        "score": 38.47133838469635
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.89447005427888
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 46.07913376963507,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 70.47783846737516
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 46.07913376963507
      },
      "fr": {
       "accuracy": 0.42857142857142855,
       "cc": 23.839275340930822,
       "cc_by_type": {
        "choice": 15.294117647058824,
        "noul": 0,
        "score": 56.223708375733636
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 23.839275340930822
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 27.037410682068597,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 0,
        "score": 31.39156165514434
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 27.037410682068597
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 67.26700899902467,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 15.830645079692863
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 67.26700899902467
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 22.223201710577673,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 3.857849770335098
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 22.223201710577673
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 49.120207596142166,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 47.36062278842651
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 49.120207596142166
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.3181021434982,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 95.62097309716124
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.3181021434982
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 0.6113352439164336,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 27.80803170577527
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.6113352439164336
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 53.9772539570996,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 57.16663279808676
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 53.9772539570996
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 67.38198878819895,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": 55.18073360089622
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 67.38198878819895
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 26.13951124323292,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 35.01427841054983
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 26.13951124323292
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 29.199516858326863,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 31.076811444545815
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 29.199516858326863
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.32623665099804,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 81.4948389852522
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.32623665099804
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 31.384878910972457,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": -13.754967786856632
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 31.384878910972457
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7693333333333333,
       "cc": 70.2060889033468,
       "cc_by_type": {
        "choice": 70.2060889033468
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 70.2060889033468
      },
      "noul": {
       "accuracy": 0.6613333333333333,
       "cc": 32.266666666666666,
       "cc_by_type": {
        "noul": 32.266666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 32.266666666666666
      },
      "score": {
       "accuracy": 0.5893333333333334,
       "cc": 42.502298232284154,
       "cc_by_type": {
        "score": 42.502298232284154
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 42.502298232284154
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 19.398592732098425,
       "cc_by_type": {
        "choice": 36,
        "noul": 0,
        "score": 22.19577819629528
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 19.398592732098425
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": 4.560003558944312,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": -22.896938943594947
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 4.560003558944312
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 54.984605803255306,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 63.73430521464396
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 54.984605803255306
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -32.94550967102398,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -28.66109041658069
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 22.84201437357555,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 45.337637323625216
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.84201437357555
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": 12.742920575752285,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": 33.33333333333333,
        "score": -19.63287349286893
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 12.742920575752285
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 36.722035568569,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 25.29215712587507
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 36.722035568569
      },
      "gaming": {
       "accuracy": 0.5454545454545454,
       "cc": 28.13714675856338,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": 0,
        "score": 36.67624863805947
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 28.13714675856338
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 18.23892289455222,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60,
        "score": 56.38343535032333
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.23892289455222
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 43.4296215704276,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 33.33333333333333,
        "score": 35.49942859422358
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 43.4296215704276
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 50.10373799055933,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 28.60578761508887
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 50.10373799055933
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 15.263876228821458,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -33.333333333333336,
        "score": 11.383026535926744
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 15.263876228821458
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 78.76271298376912,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 100,
        "score": 47.804453730577976
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 78.76271298376912
      },
      "model_routing": {
       "accuracy": 0.7880184331797235,
       "cc": 67.78953178167247,
       "cc_by_type": {
        "choice": 71.94805194805195,
        "noul": 93.93939393939394,
        "score": 37.48114945757148
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 67.78953178167247
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 11.731104036301266,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 69.74225644671569
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 11.731104036301266
      },
      "recruiting": {
       "accuracy": 0.8888888888888888,
       "cc": 85.90646185176752,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 57.719385555302516
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 85.90646185176752
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 7.194103790460911,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50,
        "score": 30.695611863993566
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 7.194103790460911
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": -11.791006340242546,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100,
        "score": 22.59799547202598
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 79.5102568122854,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 88.53077043685622
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 79.5102568122854
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.13996631431465,
      89.24571617175492
     ],
     "I": [
      44.496386091996314,
      53.73836773548576
     ],
     "capability": [
      64.62383573654232,
      70.61089490969697
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 52
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.03920345557668,
     "capability": 69.31979665405669,
     "intelligence": 49.60038985253669
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7693333333333333,
      "calibration": 93.63197131583925,
      "cc": 70.2060889033468,
      "cc_by_type": {
       "choice": 70.2060889033468
      },
      "cc_tiered": 71.14567363985879,
      "cc_tiered_open": 75.4454905515748,
      "cc_tiered_sealed": 66.84585672814276,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.2060889033468
     },
     "noul": {
      "accuracy": 0.6613333333333333,
      "calibration": 86.10910116796775,
      "cc": 32.266666666666666,
      "cc_by_type": {
       "noul": 32.266666666666666
      },
      "cc_tiered": 34.47685290013379,
      "cc_tiered_open": 44.98550724637681,
      "cc_tiered_sealed": 23.96819855389078,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 32.266666666666666
     },
     "score": {
      "accuracy": 0.5893333333333334,
      "calibration": 87.37653788292307,
      "cc": 42.502298232284154,
      "cc_by_type": {
       "score": 42.502298232284154
      },
      "cc_tiered": 43.17864301761752,
      "cc_tiered_open": 46.19024579967359,
      "cc_tiered_sealed": 40.167040235561444,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 42.502298232284154
     }
    },
    "raw_on_A": {
     "calibration": 84.05380304593929,
     "capability": 67.14117186884195,
     "intelligence": 50.228540691744605
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "adapter_id": "a3/candidates/nemotron-diffusion-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "pst2154",
   "axes": {
    "calibration": 74.53740737677316,
    "cost": 55.49170925809556,
    "intelligence": 20.38848344571935,
    "speed": 93.55093866077343
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10832159242229476,
      "ece_score": 78.33568151554104,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 78.33568151554104,
      "score_before_time_limit": 78.33568151554104,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3881401617250674,
      "brier": 0.22241599402070583,
      "ece": 0.17236678386479326,
      "n": 371,
      "score": 65.52664322704135,
      "score_before_time_limit": 65.52664322704135,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9214092140921409,
      "interval90_width": 0.6684733514001803,
      "n": 369,
      "nrps": 0.14545408533817059,
      "score": 79.74989738773714,
      "score_before_time_limit": 79.74989738773714,
      "time_limit_failed_share": 0,
      "top_ece": 0.12977398345354327
     }
    },
    "score": 74.53740737677316
   },
   "capability": 47.46294541124625,
   "class": "system-one-open",
   "composite_ci95": {
    "A": [
     4.166408924156154,
     10.200690713939366
    ],
    "B": [
     3.2795753619277317,
     8.48009811794044
    ],
    "C": [
     2.8933395306639964,
     7.083812995791227
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03045157019704434
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Nemotron Diffusion 8B (pst2154, optimized vLLM)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 22.973528132452522,
    "I_sealed": 17.803438758986175,
    "base": 20.38848344571935,
    "excess": 2.596701731028103,
    "gap": 5.170089373466347,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 45.141807097909854,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 37.72972972972973,
       "hard": 32.05410506448569,
       "judge": 48.58272907053395,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -21.739130434782606,
       "judge": -40,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 23.77877729944772,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 44.848253383055216,
       "hard": 4.46243778457357,
       "judge": 33.03604774332046,
       "standard": 37.99081262158317
      }
     },
     "sealed|choice": {
      "cc": 40.98559356961426,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 61.4219200354741,
       "hard": 25.169683257918543,
       "judge": 50.65176908752329,
       "standard": 47.89998768321223
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": -43.689320388349515,
       "judge": 5.263157894736836,
       "standard": -1.4084507042253502
      }
     },
     "sealed|score": {
      "cc": 12.424722707344273,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 42.099894800900906,
       "hard": 9.652238984201755,
       "judge": 2.7258103611165985,
       "standard": 17.6804726261925
      }
     }
    }
   },
   "jevbench_score": 7.294934989446692,
   "key": "nemotron-diffusion-8b",
   "last_measured_on": "2026-10-04",
   "licence": "MIT (application code); modified vLLM runtime Apache-2.0; nvidia/Nemotron-Labs-Diffusion-8B keeps its upstream licence",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7665198237885462,
    "decisive_rate": 0.6118598382749326,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 74.53740737677316,
     "cost": 55.49170925809556,
     "intelligence": 20.38848344571935,
     "speed": 93.8305724848098
    },
    "board_release": "v1.6.43",
    "capability": 47.46294541124625,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 97,
    "capability_rank": 124,
    "composite_ci95": {
     "A": [
      4.1676910325965215,
      10.20449077801364
     ],
     "B": [
      3.2802107600784733,
      8.482218509265811
     ],
     "C": [
      2.894229883747585,
      7.086451929176138
     ]
    },
    "jevbench_score": 7.29748476516348,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 140,
    "rank": 179,
    "ranks": {
     "A": 179,
     "B": 181,
     "C": 181,
     "capability": 183
    },
    "scores": {
     "A": 7.29748476516348,
     "B": 5.930430684235979,
     "C": 5.06769775358575
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1939013559371233,
     "p50_s_raw": 0.02195067796856165,
     "p95_s_adjusted": 0.21348231588490307,
     "p95_s_raw": 0.03174115794245154
    },
    "views": {
     "Balanced 33:33:33": 6.417819358792639,
     "Emphasis on Accuracy 60:20:20": 4.7285860819837495,
     "Emphasis on Cost 20:20:60": 7.307751777390142,
     "Emphasis on Speed 20:60:20": 8.394346515316924,
     "Intelligence only": 3.3901175711211273
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 224,
   "ranked": true,
   "ranks": {
    "A": 224,
    "B": 224,
    "C": 225,
    "capability": 222
   },
   "repo": "https://github.com/pst2154/Nemotron_Jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 7.294934989446692,
    "B": 5.929083363060404,
    "C": 5.065927076004647
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19840535297989845,
    "p50_s_raw": 0.024202676489949226,
    "p95_s_adjusted": 0.22251164717599747,
    "p95_s_raw": 0.036255823587998746
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "nvidia/Nemotron-Labs-Diffusion-8B @ 16c67f05 (unchanged weights) served by github.com/pst2154/Nemotron_Jev @ cdc36d60 on the author's digest-pinned modified vLLM runtime: BF16, candidate-only output projection, direct candidate logits, CUDA graphs, one masked forward pass per decision, nothing generated. Disclosed by the author: the inherited option-ordering rule was selected using public benchmark examples in earlier 14B experiments. Our earlier v1.4.3 run of this system used a recipe that left the base image's older 14B application in place; this v1.5 row is a fresh measurement of the 8B.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.2662042332654,
      77.36530331396067
     ],
     "I": [
      16.421783930074305,
      23.311693043206645
     ],
     "capability": [
      43.39466614794243,
      49.39379257235283
     ],
     "composite": {
      "A": [
       4.166408924156154,
       10.200690713939366
      ],
      "B": [
       3.2795753619277317,
       8.48009811794044
      ],
      "C": [
       2.8933395306639964,
       7.083812995791227
      ]
     },
     "seed": 70
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 74.53740737677316,
     "capability": 47.46294541124625,
     "intelligence": 20.38848344571935
    },
    "per_type": {
     "choice": {
      "accuracy": 0.54,
      "calibration": 78.33568151554104,
      "cc": 41.17256993454144,
      "cc_by_type": {
       "choice": 41.17256993454144
      },
      "cc_tiered": 43.06370033376206,
      "cc_tiered_open": 45.141807097909854,
      "cc_tiered_sealed": 40.98559356961426,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 41.17256993454144
     },
     "noul": {
      "accuracy": 0.464,
      "calibration": 65.52664322704135,
      "cc": -7.199999999999996,
      "cc_by_type": {
       "noul": -7.199999999999996
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4266666666666667,
      "calibration": 79.74989738773714,
      "cc": 16.44673378084317,
      "cc_by_type": {
       "score": 16.44673378084317
      },
      "cc_tiered": 18.101750003396,
      "cc_tiered_open": 23.77877729944772,
      "cc_tiered_sealed": 12.424722707344273,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 16.44673378084317
     }
    },
    "raw_on_A": {
     "calibration": 75.92799396845535,
     "capability": 48.766079474284126,
     "intelligence": 21.6041649801129
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 6.415190040822669,
    "Emphasis on Accuracy 60:20:20": 4.72772947568733,
    "Emphasis on Cost 20:20:60": 7.305706070879352,
    "Emphasis on Speed 20:60:20": 8.386254186635064,
    "Intelligence only": 3.3901175711211273
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "EryriLabs",
   "axes": {
    "calibration": 79.60239191738147,
    "cost": 86.62542366165748,
    "intelligence": 20.23194846537782,
    "speed": 54.82852093556197
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12475073319463456,
      "ece_score": 75.04985336107308,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 75.04985336107308
     },
     "noul": {
      "abstention_rate": 0.4879356568364611,
      "brier": 0.19795608560606281,
      "ece": 0.11196353518746716,
      "n": 373,
      "score": 77.60729296250656
     },
     "score": {
      "interval90_coverage": 0.9301075268817204,
      "interval90_width": 0.6832437275985659,
      "n": 372,
      "nrps": 0.1405294657671992,
      "score": 86.15002942856476,
      "top_ece": 0.06823497283075278
     }
    },
    "score": 79.60239191738147
   },
   "capability": 49.91717019137964,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 685,652 input + 0 output tokens over 1,485 decisions = 461.718518519 tokens/decision. GPU-time cost USD 0.00279138159905938/1,000; old list-price cost USD 0.00923437037037037/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00279138159905938/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.009142026666666666
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.002791381599059377
   },
   "decision_limit": null,
   "display": "Nev-2B (EryriLabs)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-15",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Sandy, the evaluator's own SHARED CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2, no AVX-512), NOT a dedicated machine and NOT a GPU: other jobs ran beside it throughout and the 1-minute load average was sampled every 60 s (measurement receipts on file) - over this row's window of 199 samples it ran 6.89 to 35.91, mean 13.06. The author's own two documented processes, unedited: llama-server -m nev-Q8_0.gguf -c 8192 --port 8080 --metrics (llama.cpp b11372, the one build the author names; --metrics is the only addition and changes no model behaviour), and the author's shim PYTHONPATH=python python -m nev.serve --backend llama-server:http://127.0.0.1:8080 --gguf nev-Q8_0.gguf on port 8090, with --backend-arg timeout=900 (the author's own documented flag, raised from the 300 s default because a loaded shared box can otherwise turn an answerable item into a false refusal). llama.cpp chose n_threads=6, its own default; the author passes none. One request at a time from the unchanged runner. Isolation: a user+network namespace (unshare -Urn, loopback only - Sandy permits no privileged unshare -n), and the author's two processes additionally inside bwrap with a cleared environment, a read-only root, tmpfs /home and /tmp and read-only binds of only the GGUF, the llama.cpp build, the venv, its interpreter and the author's own python/ tree, so the sealed item directory does not exist inside that sandbox; the runner stays outside it and the author's code never sees the file. All three cuts were PROVEN before the sealed input was read (only lo, no routes, a name probe and a bare-IP probe both failing, and an ls of the sealed tree failing inside the sandbox) and the network cut was re-proven immediately before the first sealed request, with a synthetic smoke gate passed in between (under the pre-registration). 1,485/1,500 answered, the 15 refusals all HTTP 502 relaying llama-server's own 400 exceed_context_size_error. Raw p50 7.02 s / p95 15.29 s / max 56.88 s - this is the shared box, not the model, and it is not comparable to the GPU and API rows of this draw.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 23.36677716577721,
    "I_sealed": 17.09711976497843,
    "base": 20.23194846537782,
    "excess": -3.5698693695569297,
    "gap": 6.269657400798781,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 44.315480566561476,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 64.97297297297297,
       "hard": 43.37842088707141,
       "judge": 36.71720500988794,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -4.347826086956519,
       "judge": -30.000000000000004,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 25.784850930770162,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 55.23566077335427,
       "hard": 18.910462877818656,
       "judge": 28.909830224325873,
       "standard": 20.120753175047547
      }
     },
     "sealed|choice": {
      "cc": 42.28746532429039,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 61.88055908513342,
       "hard": 31.30565514051115,
       "judge": 47.992164544564176,
       "standard": 45.89748998101666
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -13.793103448275868,
       "hard": -14.54545454545455,
       "judge": -32.30769230769231,
       "standard": -4.477611940298509
      }
     },
     "sealed|score": {
      "cc": 9.003893970644905,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 8.190773300943444,
       "hard": 5.502957884946635,
       "judge": 6.841648353006702,
       "standard": 19.655694903349474
      }
     }
    }
   },
   "jevbench_score": 7.136510525770009,
   "key": "nev-2b",
   "last_measured_on": "2026-10-15",
   "licence": "Apache-2.0 (weights and shim); base Qwen/Qwen3.5-2B-Base Apache-2.0",
   "listing": "remeasure-pending",
   "measured_in": "v1.6.22",
   "measurement_date_status": "measured",
   "method": "v1.6",
   "model_pin": "EryriLabs/Nev-2B-GGUF@65d329f676d75cc538346e2687bed71187ae76b2",
   "not_ranked_because": "Re-measure pending under the 10 s rule: this row's per-decision times were recorded on a loaded CPU host, so they cannot be scored under the time limit. Shown with its v1.6 values until it is measured again.",
   "noul_decisive": {
    "acc_among_decisive": 0.8324607329842932,
    "decisive": 191,
    "decisive_rate": 0.5120643431635389,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.60239191738147,
     "cost": 71.03778132505249,
     "intelligence": 20.23194846537782,
     "speed": 54.82852093556197
    },
    "board_release": "v1.6.43",
    "capability": 49.91717019137964,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      3.909429912465545,
      10.019247475616854
     ],
     "B": [
      3.09621130589029,
      8.449477612845056
     ],
     "C": [
      2.714881883656628,
      6.957810746956148
     ]
    },
    "jevbench_score": 6.944821492706128,
    "measured_in": "v1.6.22",
    "method": "v1.6",
    "open_board_rank": 141,
    "rank": 181,
    "ranks": {
     "A": 181,
     "B": 183,
     "C": 182,
     "capability": 174
    },
    "scores": {
     "A": 6.944821492706128,
     "B": 5.695775807883039,
     "C": 4.822792703268145
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 12.836194814974442,
     "p50_s_raw": 6.343097407487221,
     "p95_s_adjusted": 25.627816557881303,
     "p95_s_raw": 12.738908278940652
    },
    "views": {
     "Balanced 33:33:33": 6.009092360003425,
     "Emphasis on Accuracy 60:20:20": 4.533120029666712,
     "Emphasis on Cost 20:20:60": 7.449403907098574,
     "Emphasis on Speed 20:60:20": 6.924924518443358,
     "Intelligence only": 3.3126314570949202
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/EryriLabs/Nev-2B-GGUF",
   "scores": {
    "A": 7.136510525770009,
    "B": 5.797956321577559,
    "C": 4.9559100873402855
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "n": 124,
    "p50_s_adjusted": 12.836194814974442,
    "p50_s_raw": 6.343097407487221,
    "p95_s_adjusted": 25.627816557881303,
    "p95_s_raw": 12.738908278940652
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 46.938355311096586,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": -25.851600733376934
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 46.938355311096586
      },
      "calibration_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 6.186350760967346,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": -12,
        "score": -2.6328883321562913
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 6.186350760967346
      },
      "composite_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 0.4670486607000252,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": -33.333333333333336,
        "score": -0.2248702780625278
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.4670486607000252
      },
      "fairness": {
       "accuracy": 0.5625,
       "cc": 29.939866445263462,
       "cc_by_type": {
        "choice": 100,
        "noul": -19.999999999999996,
        "score": 9.819599335790386
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 29.939866445263462
      },
      "jaggedness": {
       "accuracy": 0.29444444444444445,
       "cc": -15.553356623470975,
       "cc_by_type": {
        "choice": 14.82194417709337,
        "noul": -62.16216216216216,
        "score": 0.6801481146558652
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.5814696485623003,
       "cc": 35.257392136280366,
       "cc_by_type": {
        "choice": 48.48991811217751,
        "noul": 21.649484536082465,
        "score": 35.632773760581124
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 35.257392136280366
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -10.080419559021554,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -50,
        "score": -6.014454553353321
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5854545454545454,
       "cc": 27.40677901913386,
       "cc_by_type": {
        "choice": 57.93935441799805,
        "noul": -2.777777777777779,
        "score": 27.058760417181315
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 27.40677901913386
      },
      "negation_pair_gen": {
       "accuracy": 0.29069767441860467,
       "cc": -24.37540474381098,
       "cc_by_type": {
        "choice": 23.333333333333332,
        "noul": -86.66666666666667,
        "score": -9.792880898099598
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.5208333333333334,
       "cc": 38.748335552596544,
       "cc_by_type": {
        "choice": 38.748335552596544
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 38.748335552596544
      },
      "perturb_pair": {
       "accuracy": 0.578125,
       "cc": 29.40536694686776,
       "cc_by_type": {
        "choice": 45.94594594594596,
        "noul": 19.999999999999996,
        "score": 22.270154894657324
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 29.40536694686776
      },
      "usecase": {
       "accuracy": 0.4909090909090909,
       "cc": 15.280162860434373,
       "cc_by_type": {
        "choice": 53.43137254901962,
        "noul": -25,
        "score": 17.4091160322835
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 15.280162860434373
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -1.277013365806873,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0,
        "score": -27.640563906944426
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": -10.526315789473685,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 48.03437833196489,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 31.50712885150143
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 48.03437833196489
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 64.63827546988519,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 42.8086317193901
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 64.63827546988519
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 12.633202749033721,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -66.66666666666667,
        "score": 88.36515759533208
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 12.633202749033721
      },
      "en": {
       "accuracy": 0.45438748913987836,
       "cc": 8.999756883752474,
       "cc_by_type": {
        "choice": 38.93167034817731,
        "noul": -21.58273381294964,
        "score": 9.650334116029747
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 8.999756883752474
      },
      "es": {
       "accuracy": 0.6774193548387096,
       "cc": 41.361890062571156,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 50,
        "score": -6.338473368567121
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 41.361890062571156
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -12.72873038858315,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": -6.492202094711197
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5,
       "cc": 31.924464844536768,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0,
        "score": 59.30280629831618
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 31.924464844536768
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 10.330713957506195,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25,
        "score": -10.488305054855719
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 10.330713957506195
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8,
       "cc": 68.38283399191003,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 19.178120058348945
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 68.38283399191003
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 29.951924796051298,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 27.04401902675596
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 29.951924796051298
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 26.885351406385215,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 25.100498663600103
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 26.885351406385215
      },
      "mixed": {
       "accuracy": 0.7142857142857143,
       "cc": 61.38675762936513,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 33.33333333333333,
        "score": 84.1602728880954
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 61.38675762936513
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 5.740053731306311,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 2.501546475304217
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 5.740053731306311
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 35.205060203589944,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 0,
        "score": 20.850051537557825
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 35.205060203589944
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 8.889197461537425,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": -19.999999999999996,
        "score": 20.91626651543136
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 8.889197461537425
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": -10.185723298219905,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 6.038574786191342
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -19.83194292150323,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -50,
        "score": -22.53930702537925
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 86.98909963559849,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 60.967298906795484
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 86.98909963559849
      },
      "zh": {
       "accuracy": 0.6,
       "cc": -3.638185596880632,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -52.15749464374924
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.56,
       "cc": 43.1676840352858,
       "cc_by_type": {
        "choice": 43.1676840352858
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 43.1676840352858
      },
      "noul": {
       "accuracy": 0.424,
       "cc": -15.200000000000003,
       "cc_by_type": {
        "noul": -15.200000000000003
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.4,
       "cc": 13.212269569733326,
       "cc_by_type": {
        "score": 13.212269569733326
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 13.212269569733326
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": 6.118021937826218,
       "cc_by_type": {
        "choice": 52,
        "noul": -50,
        "score": 16.35406581347866
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 6.118021937826218
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": -7.611128605515098,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -77.77777777777779,
        "score": -7.788527293425895
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 45.204629089096905,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 18.134212470542753
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 45.204629089096905
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 11.222937880038742,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -33.333333333333336,
        "score": 37.177585569940796
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 11.222937880038742
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 27.98629504757422,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 31.784972099244424
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 27.98629504757422
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -25.92255453318933,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -33.333333333333336,
        "score": -68.96263215302712
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 20.969729542778605,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -42.85714285714286,
        "score": 25.37417462273358
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 20.969729542778605
      },
      "gaming": {
       "accuracy": 0.22727272727272727,
       "cc": -20.360993700444038,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100,
        "score": 22.54071227845883
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.2,
       "cc": -2.256037962010261,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -19.999999999999996,
        "score": 17.398552780635878
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": 3.402882822738425,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 0,
        "score": 0.1444086395214872
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 3.402882822738425
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 5.5660152900999575,
       "cc_by_type": {
        "choice": 41.860465116279066,
        "noul": -11.111111111111116,
        "score": -14.05130813486808
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 5.5660152900999575
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -12.359465939103076,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100,
        "score": -4.820333301180191
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 14.8266510341226,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -60,
        "score": 27.512582660909047
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 14.8266510341226
      },
      "model_routing": {
       "accuracy": 0.5852534562211982,
       "cc": 32.503636070558116,
       "cc_by_type": {
        "choice": 54.41558441558442,
        "noul": 18.181818181818187,
        "score": 24.91350561427175
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 32.503636070558116
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 26.903819630623776,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 0,
        "score": 26.77671800895386
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 26.903819630623776
      },
      "recruiting": {
       "accuracy": 0.5,
       "cc": 5.445115065352193,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": -33.333333333333336,
        "score": -0.19204570459336878
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 5.445115065352193
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -8.059756459304968,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50,
        "score": -15.06596888530407
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 23.630586887314262,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 47.70335486484133
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 23.630586887314262
      },
      "search_retrieval": {
       "accuracy": 0.625,
       "cc": 48.89395421647513,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": 50,
        "score": 28.710332400315085
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 48.89395421647513
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      73.63938162121107,
      82.2058545153825
     ],
     "I": [
      16.166290591940747,
      23.47078252757484
     ],
     "capability": [
      46.01620516917586,
      52.03886983180177
     ],
     "composite": null,
     "seed": 53
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.60239191738147,
     "capability": 49.91717019137964,
     "intelligence": 20.23194846537782
    },
    "per_type": {
     "choice": {
      "accuracy": 0.56,
      "calibration": 75.04985336107308,
      "cc": 43.1676840352858,
      "cc_by_type": {
       "choice": 43.1676840352858
      },
      "cc_tiered": 43.30147294542593,
      "cc_tiered_open": 44.315480566561476,
      "cc_tiered_sealed": 42.28746532429039,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 43.1676840352858
     },
     "noul": {
      "accuracy": 0.424,
      "calibration": 77.60729296250656,
      "cc": -15.200000000000003,
      "cc_by_type": {
       "noul": -15.200000000000003
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4,
      "calibration": 86.15002942856476,
      "cc": 13.212269569733326,
      "cc_by_type": {
       "score": 13.212269569733326
      },
      "cc_tiered": 17.394372450707532,
      "cc_tiered_open": 25.784850930770162,
      "cc_tiered_sealed": 9.003893970644905,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 13.212269569733326
     }
    },
    "raw_on_A": {
     "calibration": 77.01332672896311,
     "capability": 47.71286883151212,
     "intelligence": 18.412410934061132
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 6.201258889607662,
    "Emphasis on Accuracy 60:20:20": 4.59760660029811,
    "Emphasis on Cost 20:20:60": 8.002784815958506,
    "Emphasis on Speed 20:60:20": 7.076551671884466,
    "Intelligence only": 3.3126314570949202
   }
  },
  {
   "adapter_id": "images/nimble-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bespoke Labs",
   "axes": {
    "calibration": 69.88762704651819,
    "cost": 56.42334655643978,
    "intelligence": 43.14237010986392,
    "speed": 85.53966579201045
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.178988177870932,
      "ece_score": 64.2023644258136,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 64.2023644258136,
      "score_before_time_limit": 64.2023644258136,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2264150943396226,
      "brier": 0.14509070919724038,
      "ece": 0.1305339256295796,
      "n": 371,
      "score": 73.89321487408408,
      "score_before_time_limit": 73.89321487408408,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8401084010840109,
      "interval90_width": 0.26874435411020764,
      "n": 369,
      "nrps": 0.11567566172039558,
      "score": 71.56730183965689,
      "score_before_time_limit": 71.56730183965689,
      "time_limit_failed_share": 0,
      "top_ece": 0.22648915074323328
     }
    },
    "score": 69.88762704651819
   },
   "capability": 56.514998578191054,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,079,172 input + 3,248 output tokens over 1,624 decisions = 1282.27832512 tokens/decision. GPU-time cost USD 0.0283501339224056/1,000; old list-price cost USD 0.128327832512315/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0283501339224056/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02835013392240557
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bespoke Nimble 9B (Bespoke Labs)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 46.96255933707536,
    "I_sealed": 39.32218088265248,
    "base": 43.14237010986392,
    "excess": 5.0669908119846365,
    "gap": 7.64037845442288,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 59.86495466898259,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 49.040578798364265,
       "judge": 68.35860250494396,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 38.9399585921325,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 47.82608695652173,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 42.08276475011101,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.18206312365201,
       "hard": 44.02495901721014,
       "judge": 36.708929948504554,
       "standard": 43.70947923155191
      }
     },
     "sealed|choice": {
      "cc": 57.35467556665829,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 60.0916414160077,
       "hard": 46.5497737556561,
       "judge": 60.89385474860336,
       "standard": 72.28722749107035
      }
     },
     "sealed|noul": {
      "cc": 27.490153799650226,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 52,
       "hard": 4.854368932038833,
       "judge": 36.8421052631579,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 33.12171328164895,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 49.98365384887967,
       "hard": 27.146627400052527,
       "judge": 32.82017491999325,
       "standard": 37.093222303709986
      }
     }
    }
   },
   "jevbench_score": 44.51360799306354,
   "key": "nimble-9b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (weights); repository without a licence file as of 19 Sep",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.867595818815331,
    "decisive_rate": 0.7735849056603774,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 69.88762704651819,
     "cost": 36.75037423027614,
     "intelligence": 43.14237010986392,
     "speed": 86.11327010751721
    },
    "board_release": "v1.6.43",
    "capability": 56.514998578191054,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      15.043066969876142,
      26.052957246349575
     ],
     "B": [
      14.110886674014932,
      25.283661649629934
     ],
     "C": [
      10.446574284636212,
      18.092331421076096
     ]
    },
    "jevbench_score": 21.082616704957896,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 90,
    "rank": 124,
    "ranks": {
     "A": 124,
     "B": 127,
     "C": 137,
     "capability": 157
    },
    "scores": {
     "A": 21.082616704957896,
     "B": 20.213517776427228,
     "C": 14.64070604510965
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.43987150778993966,
     "p50_s_raw": 0.14493575389496982,
     "p95_s_adjusted": 0.5563488021492958,
     "p95_s_raw": 0.2031744010746479
    },
    "views": {
     "Balanced 33:33:33": 19.46099526058868,
     "Emphasis on Accuracy 60:20:20": 18.55883910946502,
     "Emphasis on Cost 20:20:60": 17.273520313490142,
     "Emphasis on Speed 20:60:20": 23.596180145578664,
     "Intelligence only": 17.35223877358268
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 131,
   "ranked": true,
   "ranks": {
    "A": 131,
    "B": 134,
    "C": 137,
    "capability": 189
   },
   "repo": "https://github.com/bespokelabsai/nimble",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 44.51360799306354,
    "B": 41.324477320869185,
    "C": 30.91222777296079
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.46431010039523246,
    "p50_s_raw": 0.15715505019761622,
    "p95_s_adjusted": 0.601485676318407,
    "p95_s_raw": 0.2257428381592035
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "bespokelabs/Bespoke-Nimble-9B (LoRA, adapter unchanged since 93ec5d6), merged into Qwen/Qwen3.5-9B@c202236 with the author's PEFT safe-merge",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      65.1956012319158,
      73.91457472170808
     ],
     "I": [
      37.37380617272022,
      47.11791102705384
     ],
     "capability": [
      52.152502588110366,
      59.801663230716926
     ],
     "composite": null,
     "seed": 71
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 69.88762704651819,
     "capability": 56.514998578191054,
     "intelligence": 43.14237010986392
    },
    "per_type": {
     "choice": {
      "accuracy": 0.668,
      "calibration": 64.2023644258136,
      "cc": 57.541941778842954,
      "cc_by_type": {
       "choice": 57.541941778842954
      },
      "cc_tiered": 58.60981511782044,
      "cc_tiered_open": 59.86495466898259,
      "cc_tiered_sealed": 57.35467556665829,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.541941778842954
     },
     "noul": {
      "accuracy": 0.664,
      "calibration": 73.89321487408408,
      "cc": 32.800000000000004,
      "cc_by_type": {
       "noul": 32.800000000000004
      },
      "cc_tiered": 33.215056195891364,
      "cc_tiered_open": 38.9399585921325,
      "cc_tiered_sealed": 27.490153799650226,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 32.800000000000004
     },
     "score": {
      "accuracy": 0.536,
      "calibration": 71.56730183965689,
      "cc": 35.587315474154515,
      "cc_by_type": {
       "score": 35.587315474154515
      },
      "cc_tiered": 37.60223901587998,
      "cc_tiered_open": 42.08276475011101,
      "cc_tiered_sealed": 33.12171328164895,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.587315474154515
     }
    },
    "raw_on_A": {
     "calibration": 72.02241547589321,
     "capability": 59.4874434751425,
     "intelligence": 46.95247147439178
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 42.46818258514933,
    "Emphasis on Accuracy 60:20:20": 37.61997124185288,
    "Emphasis on Cost 20:20:60": 42.282702211609084,
    "Emphasis on Speed 20:60:20": 48.99760389900812,
    "Intelligence only": 32.11973784726633
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Bespoke Labs",
   "axes": {
    "calibration": 86.63426567315432,
    "cost": 60.74282184772803,
    "intelligence": 41.405096245385266,
    "speed": 88.52559981079375
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08196727615205182,
      "ece_score": 83.60654476958965,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 83.60654476958965,
      "score_before_time_limit": 83.60654476958965,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.42666666666666664,
      "brier": 0.11036389023557593,
      "ece": 0.06483675752018682,
      "n": 375,
      "score": 87.03264849596263,
      "score_before_time_limit": 87.03264849596263,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.984,
      "interval90_width": 0.5255555555555554,
      "n": 375,
      "nrps": 0.08473015745164805,
      "score": 89.26360375391067,
      "score_before_time_limit": 89.26360375391067,
      "time_limit_failed_share": 0,
      "top_ece": 0.0649988837350692
     }
    },
    "score": 86.63426567315432
   },
   "capability": 64.01968095926979,
   "composite_ci95": {
    "A": [
     32.26129596334245,
     56.1695300004645
    ],
    "B": [
     28.703385183144004,
     51.85751081932692
    ],
    "C": [
     22.403677752321148,
     39.00661805587812
    ]
   },
   "cost": {
    "basis": "Bespoke Labs public list price, USD 0.04/M input, output free (fetched 16 Oct 2026): Bespoke's Nimble pricing page (docs.bespokelabs.ai/nimble/pricing, saved copy sha256 03b06261...) says \"Nimble charges for input tokens only. Output tokens are free.\" and lists nimble-v4 at USD 0.04 per 1M input tokens. Counted as the manufacturer's standard, non-promotional launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap and d1-ultra), through a scoped re-freeze of the frozen pricing module that adds exactly three keys (sage-1.4.0, nimble-v4, nimble-v4-max) to its launch-list-price set and nothing else, with a zero-difference proof for every other registry entry and the independent release review; it is the separate pricing change made the cost axis wait for. Free credit for new accounts is a trial credit and is not used: the row is priced at the published tariff, never as cost 0. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (1,935,351 input tokens over all 1,500 answered items; the 3,000 reported output tokens are free under the tariff), divided by the 1,485 items of the common cost basis the scorer uses. USD 0.020350 per 1,000 decisions on the common basis (USD 0.051609 over all 1,500 items), 0.315x the official cost cap USD 0.064595. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.020350 per 1,000 decisions (0.315x the official cost cap USD 0.064595), against USD 0.051609 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05160936000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020350383838383838
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bespoke Nimble-v4 (Bespoke Labs)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-14",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.bespokelabs.ai /v1/systemone API, model `nimble-v4`, through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `bespoke` before the first request (every sealed item of this draw was already retired). Native probabilities on all three question types. Hosting implementation and deployed weights were not independently inspected. Measured as our own add after Bespoke Labs' public announcement of 13 Oct 2026 (no paid order); key on Benchmark Heaven's own account with Bespoke's free credit.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 46.227471634307484,
    "I_sealed": 36.582720856463055,
    "base": 41.405096245385266,
    "excess": -0.19477599251128197,
    "gap": 9.644750777844429,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 74.38459954164753,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 69.80182447310474,
       "judge": 76.26895187870798,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 13.326432022084196,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 4.347826086956519,
       "judge": 0,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 50.97138333919072,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.922961492763804,
       "hard": 58.84686036423604,
       "judge": 45.799744250773585,
       "standard": 43.502098844939226
      }
     },
     "sealed|choice": {
      "cc": 66.57364668901572,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 82.84625158831004,
       "hard": 66.69365097721753,
       "judge": 57.68854064642509,
       "standard": 71.52499472685088
      }
     },
     "sealed|noul": {
      "cc": 3.5399803490385073,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": 3.6363636363636376,
       "judge": -19.999999999999996,
       "standard": 28.358208955223873
      }
     },
     "sealed|score": {
      "cc": 39.63453553133493,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 63.47008501291492,
       "hard": 32.35700189778908,
       "judge": 34.84728157786855,
       "standard": 49.4527089878362
      }
     }
    }
   },
   "jevbench_score": 43.22874272615112,
   "key": "nimble-v4",
   "last_measured_on": "2026-10-14",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "listing_note": "Bespoke told us in a chat on 16 Oct 2026 that Nimble-v4 is deprecated; its pricing page still lists it (only nimble-v3 is marked deprecated there). The page lists it at USD 0.04 per million input tokens. Measured values, price and rank unchanged (JevBench v1.6.26 notes).",
   "measured_in": "v1.6.18",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.bespokelabs.ai/v1/systemone#nimble-v4",
   "noul_decisive": {
    "acc_among_decisive": 0.9627906976744186,
    "decisive": 215,
    "decisive_rate": 0.5733333333333334,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 26,
    "axes": {
     "calibration": 86.63426567315432,
     "cost": 60.74282184772803,
     "intelligence": 41.405096245385266,
     "speed": 88.39856712562789
    },
    "board_release": "v1.6.43",
    "capability": 64.01968095926979,
    "capability_api_board_rank": 22,
    "capability_open_board_rank": null,
    "capability_rank": 63,
    "composite_ci95": {
     "A": [
      32.253505531154396,
      56.15459520353683
     ],
     "B": [
      28.698448652413568,
      51.84732628344259
     ],
     "C": [
      22.39826772996833,
      38.9962466691228
     ]
    },
    "jevbench_score": 43.21768647452467,
    "measured_in": "v1.6.18",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 74,
    "ranks": {
     "A": 74,
     "B": 78,
     "C": 78,
     "capability": 105
    },
    "scores": {
     "A": 43.21768647452467,
     "B": 39.13164408365893,
     "C": 30.01228227397547
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.23237984799197875,
     "p50_s_raw": 0.23237984799197875,
     "p95_s_adjusted": 0.6222212287582807,
     "p95_s_raw": 0.6222212287582807
    },
    "views": {
     "Balanced 33:33:33": 39.618380485230354,
     "Emphasis on Accuracy 60:20:20": 34.208925086970844,
     "Emphasis on Cost 20:20:60": 40.408487852374435,
     "Emphasis on Speed 20:60:20": 45.99178065109866,
     "Intelligence only": 28.393660603216478
    }
   },
   "rank": 138,
   "ranked": true,
   "ranks": {
    "A": 138,
    "B": 143,
    "C": 143,
    "capability": 135
   },
   "repo": "https://bespokelabs.ai",
   "repriced_in": "v1.6.23",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 43.22874272615112,
    "B": 39.13889512595354,
    "C": 30.019960226493836
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2323382355098147,
    "p50_s_raw": 0.2323382355098147,
    "p95_s_adjusted": 0.6043928697079537,
    "p95_s_raw": 0.6043928697079537
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 62.87504268080306,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 5.291794709075859
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 62.87504268080306
      },
      "calibration_gen": {
       "accuracy": 0.5544554455445545,
       "cc": 36.71840534927043,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 36.00000000000001,
        "score": 21.875258465626757
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 36.71840534927043
      },
      "composite_gen": {
       "accuracy": 0.6831683168316832,
       "cc": 38.05576935546986,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": -25.92592592592593,
        "score": 46.597299032985916
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 38.05576935546986
      },
      "fairness": {
       "accuracy": 0.8333333333333334,
       "cc": 68.76166602198055,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 72.95166473260831
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 68.76166602198055
      },
      "jaggedness": {
       "accuracy": 0.5444444444444444,
       "cc": 22.243715441838727,
       "cc_by_type": {
        "choice": 46.58325312800771,
        "noul": -29.729729729729726,
        "score": 49.877622927238185
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 22.243715441838727
      },
      "llm_router": {
       "accuracy": 0.7603833865814696,
       "cc": 62.50547658592752,
       "cc_by_type": {
        "choice": 66.71656247248393,
        "noul": 62.88659793814433,
        "score": 57.91326934715432
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 62.50547658592752
      },
      "long_state": {
       "accuracy": 0.5333333333333333,
       "cc": 4.871919127970649,
       "cc_by_type": {
        "choice": 61.29032258064516,
        "noul": 0,
        "score": -46.67456519673321
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 4.871919127970649
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -12.631364067994236,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -75,
        "score": -13.409556121508492
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6581818181818182,
       "cc": 36.79679245547057,
       "cc_by_type": {
        "choice": 73.58982719269645,
        "noul": -11.111111111111116,
        "score": 47.91166128482639
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 36.79679245547057
      },
      "negation_pair_gen": {
       "accuracy": 0.627906976744186,
       "cc": 17.910040456548206,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": -46.666666666666664,
        "score": 7.063454702977956
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 17.910040456548206
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 71.46060305149378,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100,
        "score": 44.41183918451137
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 71.46060305149378
      },
      "usecase": {
       "accuracy": 0.5727272727272728,
       "cc": 30.241095001794505,
       "cc_by_type": {
        "choice": 62.00980392156864,
        "noul": -8.333333333333337,
        "score": 37.04681441714821
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 30.241095001794505
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": -0.4063065270671255,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 0,
        "score": -63.12368148596328
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 17.33676891316524,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 6.0809672617691435
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 17.33676891316524
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 72.84971754245485,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 67.44295793709905
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 72.84971754245485
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 33.173708716426326,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -66.66666666666667,
        "score": 99.70734588857135
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 33.173708716426326
      },
      "en": {
       "accuracy": 0.6637706342311034,
       "cc": 42.06591187806426,
       "cc_by_type": {
        "choice": 68.58715417190416,
        "noul": 13.669064748201443,
        "score": 43.94151671408719
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 42.06591187806426
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 67.31271336713421,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 11.726068323262318
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.31271336713421
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 25.707953232257534,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -33.333333333333336,
        "score": 43.24407827600757
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 25.707953232257534
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 72.26504569240879,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 80.32454884193226
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 72.26504569240879
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 22.236011151779042,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25,
        "score": 25.22758652796282
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 22.236011151779042
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 71.53128312800176,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 28.623467466624174
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.53128312800176
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 14.784765883400851,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -19.999999999999996,
        "score": 21.542542288804622
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 14.784765883400851
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 10.452983327342158,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 31.358949982026463
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 10.452983327342158
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.30841577705657,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 98.59191399783639
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.30841577705657
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -6.49782092272594,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 32.45458917987413
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 38.61167387134184,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 51.06989254081351
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 38.61167387134184
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 50.52489539329753,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 19.999999999999996,
        "score": 63.39554652167219
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 50.52489539329753
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 13.221329392617008,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 36.25973285870208
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 13.221329392617008
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 37.8682340701448,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50,
        "score": -14.65616735478299
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 37.8682340701448
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 80.86133582192467,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 42.584007465774
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 80.86133582192467
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 11.860785606137426,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -31.08430984825439
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 11.860785606137426
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7626666666666667,
       "cc": 69.34499320691174,
       "cc_by_type": {
        "choice": 69.34499320691174
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 69.34499320691174
      },
      "noul": {
       "accuracy": 0.552,
       "cc": 10.40000000000001,
       "cc_by_type": {
        "noul": 10.40000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 10.40000000000001
      },
      "score": {
       "accuracy": 0.576,
       "cc": 43.476768622445704,
       "cc_by_type": {
        "score": 43.476768622445704
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 43.476768622445704
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 2.2786515839975863,
       "cc_by_type": {
        "choice": 68,
        "noul": -75,
        "score": 13.835954751992762
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 2.2786515839975863
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 16.217833358888537,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -11.111111111111116,
        "score": -2.968308066881664
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 16.217833358888537
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 30.601303418460095,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": 40.58439806025833
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 30.601303418460095
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -29.26143997524909,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100,
        "score": 5.782931536241042
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 22.322772125960967,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 58.27266420396986
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.322772125960967
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -22.96464494240405,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -66.66666666666667,
        "score": -26.75557004733793
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 17.40028973466517,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -42.85714285714286,
        "score": 14.665855198393274
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 17.40028973466517
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 2.0356283791569916,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -100,
        "score": 37.465770154892574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 2.0356283791569916
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": 5.2997131864851355,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 38.399139559455406
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 5.2997131864851355
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 41.33594739962141,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 33.33333333333333,
        "score": 29.218406081805014
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 41.33594739962141
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 24.531777745700577,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": 18.556573547179266
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 24.531777745700577
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 12.053690294805428,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -33.333333333333336,
        "score": 1.7524687338786538
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 12.053690294805428
      },
      "llm_guardrails": {
       "accuracy": 0.7037037037037037,
       "cc": 41.77608289103801,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -19.999999999999996,
        "score": 56.84456345238465
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 41.77608289103801
      },
      "model_routing": {
       "accuracy": 0.815668202764977,
       "cc": 68.58454886246625,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 81.81818181818181,
        "score": 44.974425808178
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 68.58454886246625
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 8.881552486989458,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 61.193601798780286
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 8.881552486989458
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 50.38160882884504,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 51.23767699721296
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.38160882884504
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -0.19458016484627372,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50,
        "score": 8.529559998072012
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 13.047825366682751,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 30.447823926135207
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 13.047825366682751
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 93.19059202541666,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 79.57177607624996
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 93.19059202541666
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.37908894972551,
      87.49271267670296
     ],
     "I": [
      36.80192764711807,
      46.306385990881296
     ],
     "capability": [
      60.20595188468623,
      65.72729625625716
     ],
     "composite": {
      "A": [
       32.26129596334245,
       56.1695300004645
      ],
      "B": [
       28.703385183144004,
       51.85751081932692
      ],
      "C": [
       22.403677752321148,
       39.00661805587812
      ]
     },
     "seed": 54
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.63426567315432,
     "capability": 64.01968095926979,
     "intelligence": 41.405096245385266
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7626666666666667,
      "calibration": 83.60654476958965,
      "cc": 69.34499320691174,
      "cc_by_type": {
       "choice": 69.34499320691174
      },
      "cc_tiered": 70.47912311533162,
      "cc_tiered_open": 74.38459954164753,
      "cc_tiered_sealed": 66.57364668901572,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.34499320691174
     },
     "noul": {
      "accuracy": 0.552,
      "calibration": 87.03264849596263,
      "cc": 10.40000000000001,
      "cc_by_type": {
       "noul": 10.40000000000001
      },
      "cc_tiered": 8.433206185561351,
      "cc_tiered_open": 13.326432022084196,
      "cc_tiered_sealed": 3.5399803490385073,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 10.40000000000001
     },
     "score": {
      "accuracy": 0.576,
      "calibration": 89.26360375391067,
      "cc": 43.476768622445704,
      "cc_by_type": {
       "score": 43.476768622445704
      },
      "cc_tiered": 45.30295943526283,
      "cc_tiered_open": 50.97138333919072,
      "cc_tiered_sealed": 39.63453553133493,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.476768622445704
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 39.630769642234846,
    "Emphasis on Accuracy 60:20:20": 34.21446640379054,
    "Emphasis on Cost 20:20:60": 40.41621985641152,
    "Emphasis on Speed 20:60:20": 46.021843440308366,
    "Intelligence only": 28.393660603216478
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Bespoke Labs",
   "axes": {
    "calibration": 91.24998737212707,
    "cost": 60.74282184772803,
    "intelligence": 68.18491883484498,
    "speed": 87.92708417376494
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04168013967750483,
      "ece_score": 91.66397206449903,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 91.66397206449903,
      "score_before_time_limit": 91.66397206449903,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.21599999999999997,
      "brier": 0.060518846834681644,
      "ece": 0.04479113991582226,
      "n": 375,
      "score": 91.04177201683554,
      "score_before_time_limit": 91.04177201683554,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9973333333333333,
      "interval90_width": 0.3084444444444445,
      "n": 375,
      "nrps": 0.04645325205782863,
      "score": 91.04421803504661,
      "score_before_time_limit": 91.04421803504661,
      "time_limit_failed_share": 0,
      "top_ece": 0.06633119362061955
     }
    },
    "score": 91.24998737212707
   },
   "capability": 79.71745310348602,
   "composite_ci95": {
    "A": [
     72.72057952799157,
     75.71656715778371
    ],
    "B": [
     70.5838602411584,
     74.87875387318591
    ],
    "C": [
     72.72057952799157,
     75.71656715778371
    ]
   },
   "cost": {
    "basis": "Bespoke Labs public list price, USD 0.04/M input, output free: Bespoke's Nimble pricing page (docs.bespokelabs.ai/nimble/pricing) says \"Nimble charges for input tokens only. Output tokens are free.\" and, in the copy fetched 16 Oct 2026 at about 21:40Z (saved copy sha256 4d46184d...), lists nimble-v4-max at USD 0.04 per 1M input tokens. The copy of the same page fetched 16 Oct 2026 at 15:58Z (saved copy sha256 03b06261...), from which v1.6.23 first priced this row, listed USD 0.09; v1.6.26 re-prices the row to the new public tariff through a scoped re-freeze of the frozen pricing module that changes the documented tariff of this one key and nothing else (no key joins or leaves its launch-list-price set; zero difference for every other registry entry). This is the current public list price on the vendor's pricing page (changed 16 Oct 2026; earlier copy listed USD 0.09), counted under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap and d1-ultra). Free credit for new accounts is a trial credit and is not used: the row is priced at the published tariff, never as cost 0. No new measurement: ESTIMATE in the scorer's sense, the tariff x this row's OWN provider-reported tokens from its v1.6.18 measurement (1,935,351 input tokens over all 1,500 answered items; the 3,000 reported output tokens are free under the tariff), divided by the 1,485 items of the common cost basis the scorer uses. USD 0.020350 per 1,000 decisions on the common basis (USD 0.051609 over all 1,500 items), 0.315x the official cost cap USD 0.064595 (at the earlier USD 0.09 tariff it was over the cap). Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.020350 per 1,000 decisions (0.315x the official cost cap USD 0.064595), against USD 0.051609 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05160936000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020350383838383838
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Bespoke Nimble-v4-max (Bespoke Labs)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-14",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.bespokelabs.ai /v1/systemone API, model `nimble-v4-max`, through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `bespoke` before the first request (every sealed item of this draw was already retired). Native probabilities on all three question types. Hosting implementation and deployed weights were not independently inspected. Measured as our own add after Bespoke Labs' public announcement of 13 Oct 2026 (no paid order); key on Benchmark Heaven's own account with Bespoke's free credit.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.20451669265262,
    "I_sealed": 64.16532097703733,
    "base": 68.18491883484498,
    "excess": -1.8003310547404183,
    "gap": 8.039195715615293,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 86.67735092678937,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 84.90091223655237,
       "judge": 92.089650626236,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 63.948240165631475,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 65.98795898553702,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.01753536790163,
       "hard": 72.99899229354084,
       "judge": 67.76239404368843,
       "standard": 48.28945159111995
      }
     },
     "sealed|choice": {
      "cc": 80.44759269360293,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 74.49982652943217,
       "judge": 81.48873653281098,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 46.774787744422326,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 43.636363636363626,
       "judge": 38.46153846153846,
       "standard": 49.25373134328359
      }
     },
     "sealed|score": {
      "cc": 65.27358249308672,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 79.33978396612804,
       "hard": 64.40573294358967,
       "judge": 63.511594612062325,
       "standard": 62.619162677096796
      }
     }
    }
   },
   "jevbench_score": 74.82115516763538,
   "key": "nimble-v4-max",
   "last_measured_on": "2026-10-14",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.18",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.bespokelabs.ai/v1/systemone#nimble-v4-max",
   "noul_decisive": {
    "acc_among_decisive": 0.9795918367346939,
    "decisive": 294,
    "decisive_rate": 0.784,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 5,
    "axes": {
     "calibration": 91.24998737212707,
     "cost": 60.74282184772803,
     "intelligence": 68.18491883484498,
     "speed": 91.45091520080334
    },
    "board_release": "v1.6.43",
    "capability": 79.71745310348602,
    "capability_api_board_rank": 7,
    "capability_open_board_rank": null,
    "capability_rank": 12,
    "composite_ci95": {
     "A": [
      73.30460656936545,
      76.34991725686463
     ],
     "B": [
      71.02324021608213,
      75.3734188553226
     ],
     "C": [
      73.30460656936545,
      76.34991725686463
     ]
    },
    "jevbench_score": 75.4395528754179,
    "measured_in": "v1.6.18",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 6,
    "ranks": {
     "A": 6,
     "B": 9,
     "C": 6,
     "capability": 24
    },
    "scores": {
     "A": 75.4395528754179,
     "B": 73.8677004961083,
     "C": 75.4395528754179
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.254514531508903,
     "p50_s_raw": 0.254514531508903,
     "p95_s_adjusted": 0.28131694759504167,
     "p95_s_raw": 0.28131694759504167
    },
    "views": {
     "Balanced 33:33:33": 71.32043947749163,
     "Emphasis on Accuracy 60:20:20": 70.03225099016612,
     "Emphasis on Cost 20:20:60": 66.67610875958445,
     "Emphasis on Speed 20:60:20": 78.20646502285324,
     "Intelligence only": 68.18491883484498
    }
   },
   "rank": 21,
   "ranked": true,
   "ranks": {
    "A": 21,
    "B": 26,
    "C": 19,
    "capability": 35
   },
   "repo": "https://bespokelabs.ai",
   "repriced_in": "v1.6.26",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.82115516763538,
    "B": 73.39253944111499,
    "C": 74.82115516763538
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2574840509914793,
    "p50_s_raw": 0.2574840509914793,
    "p95_s_adjusted": 0.6259523141590762,
    "p95_s_raw": 0.6259523141590762
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 77.93302244321468,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 42.13240066297739
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 77.93302244321468
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 48.46720156444257,
       "cc_by_type": {
        "choice": 61.82396606574761,
        "noul": 28.000000000000004,
        "score": 55.57763862758007
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 48.46720156444257
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 85.97656131741992,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 71.8411564003176
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.97656131741992
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.3952829117887,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 96.18584873536608
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.3952829117887
      },
      "jaggedness": {
       "accuracy": 0.7444444444444445,
       "cc": 52.85440527377147,
       "cc_by_type": {
        "choice": 68.23869104908566,
        "noul": 13.513513513513509,
        "score": 76.81101125871524
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 52.85440527377147
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 83.04124579094008,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 93.81443298969072,
        "score": 60.85654397104885
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.04124579094008
      },
      "long_state": {
       "accuracy": 0.6,
       "cc": 23.69771087111538,
       "cc_by_type": {
        "choice": 61.29032258064516,
        "noul": 0,
        "score": 9.802810032700982
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 23.69771087111538
      },
      "ml_usecase": {
       "accuracy": 0.5925925925925926,
       "cc": 18.542786785637407,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -25,
        "score": 30.11289643938645
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 18.542786785637407
      },
      "multilingual": {
       "accuracy": 0.7345454545454545,
       "cc": 54.53849518745235,
       "cc_by_type": {
        "choice": 76.5242908379524,
        "noul": 25,
        "score": 62.091194724404616
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 54.53849518745235
      },
      "negation_pair_gen": {
       "accuracy": 0.9302325581395349,
       "cc": 78.91939008748496,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 70.09150359578821
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 78.91939008748496
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 82.73195100155357,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 63.21086801967573
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 82.73195100155357
      },
      "usecase": {
       "accuracy": 0.7454545454545455,
       "cc": 56.032678027680525,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 29.166666666666675,
        "score": 64.66666153402196
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 56.032678027680525
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8125,
       "cc": 58.740127058626875,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": -4.73199977650034
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 58.740127058626875
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 41.85206203924473,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 12.960179973340946
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 41.85206203924473
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 82.27735534531733,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100,
        "score": 78.69047311559798
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 82.27735534531733
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 61.060090115548206,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 0,
        "score": 99.94004688295746
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 61.060090115548206
      },
      "en": {
       "accuracy": 0.8392701998262381,
       "cc": 71.15704384183026,
       "cc_by_type": {
        "choice": 83.52473120904064,
        "noul": 60.431654676258994,
        "score": 69.51474564019114
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 71.15704384183026
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 69.66571860471659,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 18.78508403600947
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 69.66571860471659
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 76.57541393410838,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 96.39290846899185
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 76.57541393410838
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 89.21722025064541,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 88.8281313401715
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 89.21722025064541
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 34.367393690812165,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 36.62173414506219
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 34.367393690812165
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 70.95270534511762,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 26.887734117971718
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 70.95270534511762
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 27.327090862326045,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 0,
        "score": 10.575394906279168
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 27.327090862326045
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 3.596678785557382,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 55.23448080111659
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 3.596678785557382
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.40723626765454,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 98.88837546963032
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.40723626765454
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -26.53328200822546,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": -27.651794076624437
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 25,
       "cc_by_type": {
        "choice": 100,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 25
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 73.1717555920901,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 67.13270223966425
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 73.1717555920901
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 65.04851477203407,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 66.96640465788181
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 65.04851477203407
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 40.482391585259116,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 19.999999999999996,
        "score": 52.51100454301141
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 40.482391585259116
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 60.52275499920906,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 31.568264997627193
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 60.52275499920906
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.19627963052231,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 81.10496792382497
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.19627963052231
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 33.346706726269964,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 33.373453512143236
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 33.346706726269964
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8613333333333333,
       "cc": 82.08920951415067,
       "cc_by_type": {
        "choice": 82.08920951415067
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 82.08920951415067
      },
      "noul": {
       "accuracy": 0.768,
       "cc": 53.6,
       "cc_by_type": {
        "noul": 53.6
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 53.6
      },
      "score": {
       "accuracy": 0.776,
       "cc": 66.16378952885489,
       "cc_by_type": {
        "score": 66.16378952885489
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 66.16378952885489
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 11.284240243103335,
       "cc_by_type": {
        "choice": 68,
        "noul": -75,
        "score": 40.85272072931001
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 11.284240243103335
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 81.22933598842444,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 62.321548337944144
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 81.22933598842444
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 21.699250411759156,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": 13.878239040155515
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 21.699250411759156
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -20.13552623419365,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -33.333333333333336,
        "score": -45.20190034000786
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 40.00228808667219,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 82.32570483972668
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 40.00228808667219
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": -25.10701220211857,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": -196.07575358748778
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 31.913037340162806,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": -42.85714285714286,
        "score": 48.400176446258726
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 31.913037340162806
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 52.57354917823071,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 57.72064753469211
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 52.57354917823071
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 15.471706603552159,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 68.91511981065648
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 15.471706603552159
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 64.00816563621142,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 66.66666666666667,
        "score": 63.901727458241695
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 64.00816563621142
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 57.96643471148216,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 52.19387777785735
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 57.96643471148216
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 19.212456270641866,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 21.07822902697937
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 19.212456270641866
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 55.60780315989738,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 58.33972425896277
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 55.60780315989738
      },
      "model_routing": {
       "accuracy": 0.9078341013824884,
       "cc": 79.1817133930041,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 93.93939393939394,
        "score": 49.4499020837742
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.1817133930041
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 15.930238569976515,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 82.33966004774146
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 15.930238569976515
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 57.68500793305302,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 73.1478743098369
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.68500793305302
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 36.52975459820843,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": 24.367588917777994
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 36.52975459820843
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 6.286419441145905,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -66.66666666666667,
        "score": 57.98969310604642
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 6.286419441145905
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.53112849752597,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.59338549257794
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.53112849752597
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.9087720581256,
      91.64106229553519
     ],
     "I": [
      62.909489146085264,
      71.90798251065188
     ],
     "capability": [
      75.86144294449328,
      81.13427659789494
     ],
     "composite": {
      "A": [
       72.72057952799157,
       75.71656715778371
      ],
      "B": [
       70.5838602411584,
       74.87875387318591
      ],
      "C": [
       72.72057952799157,
       75.71656715778371
      ]
     },
     "seed": 55
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.24998737212707,
     "capability": 79.71745310348602,
     "intelligence": 68.18491883484498
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8613333333333333,
      "calibration": 91.66397206449903,
      "cc": 82.08920951415067,
      "cc_by_type": {
       "choice": 82.08920951415067
      },
      "cc_tiered": 83.56247181019614,
      "cc_tiered_open": 86.67735092678937,
      "cc_tiered_sealed": 80.44759269360293,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 82.08920951415067
     },
     "noul": {
      "accuracy": 0.768,
      "calibration": 91.04177201683554,
      "cc": 53.6,
      "cc_by_type": {
       "noul": 53.6
      },
      "cc_tiered": 55.3615139550269,
      "cc_tiered_open": 63.948240165631475,
      "cc_tiered_sealed": 46.774787744422326,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 53.6
     },
     "score": {
      "accuracy": 0.776,
      "calibration": 91.04421803504661,
      "cc": 66.16378952885489,
      "cc_by_type": {
       "score": 66.16378952885489
      },
      "cc_tiered": 65.63077073931187,
      "cc_tiered_open": 65.98795898553702,
      "cc_tiered_sealed": 65.27358249308672,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 66.16378952885489
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.58506270446225,
    "Emphasis on Accuracy 60:20:20": 69.60500998393475,
    "Emphasis on Cost 20:20:60": 66.28872247626451,
    "Emphasis on Speed 20:60:20": 76.63066590299633,
    "Intelligence only": 68.18491883484498
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Bespoke Labs",
   "axes": {
    "calibration": 91.36294095928801,
    "cost": 57.47690696484853,
    "intelligence": 70.95116002556868,
    "speed": 84.78525949557437
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05149823990803967,
      "ece_score": 89.70035201839207,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 89.58075154903422,
      "score_before_time_limit": 89.70035201839207,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.1554959785522788,
      "brier": 0.059426603690029486,
      "ece": 0.037544038564276544,
      "n": 373,
      "score": 92.49119228714468,
      "score_before_time_limit": 92.49119228714468,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9838709677419355,
      "interval90_width": 0.21415770609318993,
      "n": 372,
      "nrps": 0.04303092545751453,
      "score": 92.01687904168512,
      "score_before_time_limit": 92.01687904168512,
      "time_limit_failed_share": 0,
      "top_ece": 0.058315746854391684
     }
    },
    "score": 91.36294095928801
   },
   "capability": 81.15705049242834,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 755,508 input + 0 output tokens over 1,485 decisions = 508.75959596 tokens/decision. GPU-time cost USD 0.0261478770682772/1,000; old list-price cost USD 0.21367903030303/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0261478770682772/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.21154224
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.026147877068277196
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Nimble-v4-max (open weights, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM (instance type 1A100.22V, location FIN-03, Finland): 1x NVIDIA A100-SXM4-80GB 80 GB, driver 580.178.04, torch 2.8.0+cu128. A different card from the 1x H100 80 GB on which most of this draw's other GPU rows were timed, so this row's Speed axis rests on different hardware (A100 is slower than H100). The author's own production engine nimble.evaluation.decision_index_engine:NimbleEngine (bespokelabs/nimble-v4-max-decision-index @ adea14289afa29dca3ff38b1fcb76e58f6add230, code/nimble/, unmodified) run in process behind a first-party HTTP shim (evaluator code; it only relabels a score answer's type, adds output_tokens 0 and turns the engine's Unsupported into HTTP 422), driven by the frozen run_v15 harness and the unchanged TypeSafe adapter. Engine options are the author's own environment.json settings, not tuned: shared_prefix on, max_tails 48, max_tokens 32,768. Weights bespokelabs/Nimble-v4-max @ 655aa7051c2ab39660193ebb70c8926d3d2cb7aa (11 safetensors shards hash-verified on the pod), base and tokenizer Qwen/Qwen3.8-27B @ 1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0. Isolation: fresh container with --network none (the cut proven from inside before the input was read), read-only root filesystem, all capabilities dropped, no credentials, Hub offline mode; 1,500 items in one pass, no resume, one sealed use. 15 items were refused with HTTP 422 because their prompts (76,358-80,516 input tokens) exceed the 32,768-token serving limit; nothing was truncated, and the refused set equals the set predicted by the offline count before the run. They count as wrong in every axis (full 1,500 denominator). The standard self-hosted x2 + 0.15 s own-server adjustment (an assumption, not measured) is applied on top of the raw p50/p95. Measured 21 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b); VM deleted afterwards (deletion verified).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 73.69513917272718,
    "I_sealed": 68.20718087841018,
    "base": 70.95116002556868,
    "excess": -4.3515684760387146,
    "gap": 5.487958294316996,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 81.35026138087967,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396664,
       "judge": 88.13447593935399,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 69.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 69.7869159716704,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.04039975193228,
       "hard": 75.99272722221137,
       "judge": 76.32465627204957,
       "standard": 48.44194112988871
      }
     },
     "sealed|choice": {
      "cc": 80.44964453054286,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 72.41817971550824,
       "judge": 82.37022526934379,
       "standard": 85.76249736342545
      }
     },
     "sealed|noul": {
      "cc": 57.42375894820568,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 47.272727272727266,
       "judge": 60.00000000000001,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 66.74813915648204,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 84.93672034384494,
       "hard": 63.11277519836356,
       "judge": 66.14788845209046,
       "standard": 65.82495253562494
      }
     }
    }
   },
   "jevbench_score": 73.75670981544037,
   "key": "nimble-v4-max-open",
   "last_measured_on": "2026-10-21",
   "licence": "CC BY-NC 4.0 (weights, non-commercial); base Qwen/Qwen3.8-27B Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "bespokelabs/Nimble-v4-max@655aa7051c2ab39660193ebb70c8926d3d2cb7aa",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9650793650793651,
    "decisive": 315,
    "decisive_rate": 0.8445040214477212,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 36,
   "ranked": true,
   "ranks": {
    "A": 36,
    "B": 28,
    "C": 31,
    "capability": 28
   },
   "repo": "https://huggingface.co/bespokelabs/Nimble-v4-max",
   "scores": {
    "A": 73.75670981544037,
    "B": 73.17798920771261,
    "C": 73.75670981544037
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4579101519997494,
    "p50_s_raw": 0.15395507599987468,
    "p95_s_adjusted": 0.725594159600314,
    "p95_s_raw": 0.287797079800157
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "bespokelabs/nimble-v4-max-decision-index @ adea14289afa29dca3ff38b1fcb76e58f6add230 (code/nimble/, unmodified); evaluator HTTP shim. MANDATORY DISCLOSURE: measured on a draw the author's hosted endpoint received on 14 Oct; weights committed 18 Oct. Second and last use by this author on this draw. This row is not an independent corroboration of the hosted row nimble-v4-max: the two rows come from the same author, on different hardware through a different prompt path, and the hosted row stays a separate row. We cannot verify that these weights are the bytes served by the hosted endpoint on 14 Oct, or that the exposed items played no role in producing them. Licence CC BY-NC 4.0 (weights, non-commercial). The 15 long items are refused (HTTP 422 from the engine's serving limit) and count as wrong in every axis (full 1,500 denominator). Output tokens are 0 on every answered item (the engine has no generate loop); kept as sent.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 77.50539758799437,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 40.8495260973164
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 77.50539758799437
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 55.722131664662925,
       "cc_by_type": {
        "choice": 65.00530222693531,
        "noul": 43.99999999999999,
        "score": 58.16109276705346
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 55.722131664662925
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 87.08324700297233,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 75.16121345697486
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 87.08324700297233
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.23953564456099,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 98.71860693368296
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.23953564456099
      },
      "jaggedness": {
       "accuracy": 0.7611111111111111,
       "cc": 60.810812329847664,
       "cc_by_type": {
        "choice": 66.79499518768047,
        "noul": 35.13513513513513,
        "score": 80.5023066667274
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 60.810812329847664
      },
      "llm_router": {
       "accuracy": 0.939297124600639,
       "cc": 86.01641161305686,
       "cc_by_type": {
        "choice": 95.24522321035484,
        "noul": 100,
        "score": 62.80401162881576
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 86.01641161305686
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 39.15407158120296,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 0,
        "score": 54.57561680546456
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 39.15407158120296
      },
      "multilingual": {
       "accuracy": 0.7454545454545455,
       "cc": 59.064742600566326,
       "cc_by_type": {
        "choice": 73.58982719269645,
        "noul": 38.888888888888886,
        "score": 64.71551172011365
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 59.064742600566326
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 79.53360362848692,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 71.93414421879409
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 79.53360362848692
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 77.78784655702943,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 80,
        "score": 68.3785546861033
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 77.78784655702943
      },
      "usecase": {
       "accuracy": 0.7772727272727272,
       "cc": 62.28583292370893,
       "cc_by_type": {
        "choice": 75.49019607843138,
        "noul": 41.66666666666667,
        "score": 69.70063602602873
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 62.28583292370893
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8125,
       "cc": 69.44413742315277,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": 27.380031317077336
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 69.44413742315277
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.5384615384615384,
       "cc": 33.095116601756565,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": 7.4266708498165634
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 33.095116601756565
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 83.20266789460061,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100,
        "score": 81.46641076344784
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.20266789460061
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.49330391035204,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 99.99946480368183
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.49330391035204
      },
      "en": {
       "accuracy": 0.841876629018245,
       "cc": 73.72062158622043,
       "cc_by_type": {
        "choice": 82.64605020685615,
        "noul": 67.62589928057554,
        "score": 70.88991527122961
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 73.72062158622043
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 69.1563675244238,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 17.25703079513109
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 69.1563675244238
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.6143991192104,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.50986402429791
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.6143991192104
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 89.90152600011213,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 90.88104858857166
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 89.90152600011213
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 51.65101054842187,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 25,
        "score": 46.71280818157846
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 51.65101054842187
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 72.48648788016195,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 31.489081723104707
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 72.48648788016195
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 31.013928391091792,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 0,
        "score": 21.6359074925764
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 31.013928391091792
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 5.767945316873512,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 61.748280395064995
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 5.767945316873512
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.74044306637369,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.88799586578774
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.74044306637369
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -35.64365064350917,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": -54.982899982475544
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 78.61170275794112,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80,
        "score": 71.06997920061133
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 78.61170275794112
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 65.31808698901088,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 67.77512130881225
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 65.31808698901088
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 56.56363949454061,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 60.75474827085587
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 56.56363949454061
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 74.1488039926556,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 44.185542412749435
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 74.1488039926556
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 74.07269722878466,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 86.73422071861204
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 74.07269722878466
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 52.22341428333251,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 23.336909516664207
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 52.22341428333251
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8533333333333334,
       "cc": 81.0558946784286,
       "cc_by_type": {
        "choice": 81.0558946784286
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.0558946784286
      },
      "noul": {
       "accuracy": 0.8106666666666666,
       "cc": 62.133333333333326,
       "cc_by_type": {
        "noul": 62.133333333333326
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 62.133333333333326
      },
      "score": {
       "accuracy": 0.768,
       "cc": 68.17088476320109,
       "cc_by_type": {
        "score": 68.17088476320109
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 68.17088476320109
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 23.9901133999504,
       "cc_by_type": {
        "choice": 84,
        "noul": -50,
        "score": 37.970340199851194
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 23.9901133999504
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 86.05864331437924,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 76.80947031580855
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 86.05864331437924
      },
      "customer_support": {
       "accuracy": 0.42857142857142855,
       "cc": 11.087714915362518,
       "cc_by_type": {
        "choice": 18.699186991869915,
        "noul": 0,
        "score": 14.563957754217638
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 11.087714915362518
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -5.432351070271447,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 33.33333333333333,
        "score": -67.7590415149079
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 42.193410468590194,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 88.8990719854807
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 42.193410468590194
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": -18.85230943189946,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": -164.7330289246292
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 43.47375956499048,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": -14.28571428571429,
        "score": 54.510914549313185
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 43.47375956499048
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 67.45799134901468,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50,
        "score": 62.8269357195179
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 67.45799134901468
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 31.16919546982632,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -60,
        "score": 74.34091974281229
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 31.16919546982632
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 71.82448200917604,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 74.50271083837747
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 71.82448200917604
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 66.73209462534784,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 56.268635297232194
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 66.73209462534784
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 24.466572747398118,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 36.84057845724812
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 24.466572747398118
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 70.42177274869952,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 62.781633025369196
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 70.42177274869952
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 81.8148451386316,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100,
        "score": 51.28869126005065
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 81.8148451386316
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 15.688592049548712,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 81.61472048645805
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 15.688592049548712
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 59.91651322465041,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 79.84239018462905
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 59.91651322465041
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 36.12263173958099,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": 23.146220341895674
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 36.12263173958099
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 38.34717338014102,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 0,
        "score": 73.0125346331767
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 38.34717338014102
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.94621362687455,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.83864088062363
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.94621362687455
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.36189364963293,
      92.35804441067566
     ],
     "I": [
      66.42332925082326,
      74.91599833906216
     ],
     "capability": [
      77.58373946084384,
      82.87729940235091
     ],
     "composite": null,
     "seed": 56
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.36294095928801,
     "capability": 81.15705049242834,
     "intelligence": 70.95116002556868
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8533333333333334,
      "calibration": 89.58075154903422,
      "cc": 81.0558946784286,
      "cc_by_type": {
       "choice": 81.0558946784286
      },
      "cc_tiered": 80.89995295571126,
      "cc_tiered_open": 81.35026138087967,
      "cc_tiered_sealed": 80.44964453054286,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.0558946784286
     },
     "noul": {
      "accuracy": 0.8106666666666666,
      "calibration": 92.49119228714468,
      "cc": 62.133333333333326,
      "cc_by_type": {
       "noul": 62.133333333333326
      },
      "cc_tiered": 63.685999556918574,
      "cc_tiered_open": 69.94824016563147,
      "cc_tiered_sealed": 57.42375894820568,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 62.133333333333326
     },
     "score": {
      "accuracy": 0.768,
      "calibration": 92.01687904168512,
      "cc": 68.17088476320109,
      "cc_by_type": {
       "score": 68.17088476320109
      },
      "cc_tiered": 68.26752756407622,
      "cc_tiered_open": 69.7869159716704,
      "cc_tiered_sealed": 66.74813915648204,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.17088476320109
     }
    },
    "raw_on_A": {
     "calibration": 89.46067847128532,
     "capability": 79.60532503979249,
     "intelligence": 69.74997160829966
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.30487676748037,
    "Emphasis on Accuracy 60:20:20": 69.95413647612028,
    "Emphasis on Cost 20:20:60": 64.03394866023392,
    "Emphasis on Speed 20:60:20": 74.76523361742814,
    "Intelligence only": 70.95116002556868
   }
  },
  {
   "adapter_id": "images/ninfer-qwen3.8-27b/spec.json:ninfer_native.NInferNativeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Igor L. / NInfer contributors",
   "axes": {
    "calibration": 84.18557302696732,
    "cost": 56.4886519334233,
    "intelligence": 51.800126951055375,
    "speed": 90.16889017946136
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09587088648606537,
      "ece_score": 80.82582270278694,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 80.82582270278694,
      "score_before_time_limit": 80.82582270278694,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.18328840970350402,
      "brier": 0.12070867053402283,
      "ece": 0.0947034093144267,
      "n": 371,
      "score": 81.05931813711467,
      "score_before_time_limit": 81.05931813711467,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.962059620596206,
      "interval90_width": 0.4135049683830172,
      "n": 369,
      "nrps": 0.07983756609019925,
      "score": 90.67157824100035,
      "score_before_time_limit": 90.67157824100035,
      "time_limit_failed_share": 0,
      "top_ece": 0.05336543454489691
     }
    },
    "score": 84.18557302696732
   },
   "capability": 67.99284998901135,
   "class": "native-logit",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 891,334 input + 0 output tokens over 1,624 decisions = 548.850985222 tokens/decision. GPU-time cost USD 0.0282083880173514/1,000; old list-price cost USD 0.230517413793103/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0282083880173514/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02820838801735135
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "NInfer Qwen3.8-27B NVFP4",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 51.8273559598382,
    "I_sealed": 51.77289794227256,
    "base": 51.800126951055375,
    "excess": -2.5189296248726003,
    "gap": 0.0544580175656435,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 68.57305975575287,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 56.59012268008808,
       "judge": 68.35860250494396,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 37.668046928916496,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 39.13043478260869,
       "judge": 30.000000000000004,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 49.24096119484526,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.871723226731845,
       "hard": 32.99494797533653,
       "judge": 63.458675062497626,
       "standard": 54.0910358164409
      }
     },
     "sealed|choice": {
      "cc": 63.99439201930662,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 70.73387037173899,
       "hard": 56.730769230769226,
       "judge": 63.68715083798884,
       "standard": 75.61276019214189
      }
     },
     "sealed|noul": {
      "cc": 42.068015862163264,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": 20.388349514563096,
       "judge": 50,
       "standard": 60.56338028169015
      }
     },
     "sealed|score": {
      "cc": 49.25628594534781,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.47448220301372,
       "hard": 39.79827642333791,
       "judge": 53.88993870966854,
       "standard": 51.612727714053584
      }
     }
    }
   },
   "jevbench_score": 66.69283025993185,
   "key": "ninfer-qwen3.8-27b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (engine and submitted 27B artifact/base)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8943894389438944,
    "decisive_rate": 0.816711590296496,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.18557302696732,
     "cost": 29.118887844204664,
     "intelligence": 51.800126951055375,
     "speed": 90.47734239273063
    },
    "board_release": "v1.6.43",
    "capability": 67.99284998901135,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      14.657219833459003,
      18.09581018423065
     ],
     "B": [
      14.41402694236696,
      18.285046805435694
     ],
     "C": [
      10.178624884346531,
      15.973963875467245
     ]
    },
    "jevbench_score": 17.71600496541158,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 107,
    "rank": 143,
    "ranks": {
     "A": 143,
     "B": 138,
     "C": 141,
     "capability": 91
    },
    "scores": {
     "A": 17.71600496541158,
     "B": 17.686352887683018,
     "C": 13.20459060190698
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26002668901346626,
     "p50_s_raw": 0.05501334450673312,
     "p95_s_adjusted": 0.3445464937365614,
     "p95_s_raw": 0.09727324686828069
    },
    "views": {
     "Balanced 33:33:33": 15.726436779262448,
     "Emphasis on Accuracy 60:20:20": 16.41495956225673,
     "Emphasis on Cost 20:20:60": 12.713872701078419,
     "Emphasis on Speed 20:60:20": 19.536094431908698,
     "Intelligence only": 17.568730575960352
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 101,
   "ranked": true,
   "ranks": {
    "A": 101,
    "B": 102,
    "C": 102,
    "capability": 117
   },
   "repo": "https://github.com/igorls/ninfer",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 66.69283025993185,
    "B": 63.06646774629605,
    "C": 49.70937417235107
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26485669197281825,
    "p50_s_raw": 0.05742834598640911,
    "p95_s_adjusted": 0.36316169262281617,
    "p95_s_raw": 0.10658084631140809
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B, NInfer NVFP4/FP8 artifact",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.72416061844064,
      85.97779338206968
     ],
     "I": [
      46.35923001218984,
      56.308300363548845
     ],
     "capability": [
      63.42455870096435,
      70.41651644718259
     ],
     "composite": null,
     "seed": 72
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.18557302696732,
     "capability": 67.99284998901135,
     "intelligence": 51.800126951055375
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7266666666666667,
      "calibration": 80.82582270278694,
      "cc": 65.04457054081448,
      "cc_by_type": {
       "choice": 65.04457054081448
      },
      "cc_tiered": 66.28372588752974,
      "cc_tiered_open": 68.57305975575287,
      "cc_tiered_sealed": 63.99439201930662,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.04457054081448
     },
     "noul": {
      "accuracy": 0.7226666666666667,
      "calibration": 81.05931813711467,
      "cc": 44.53333333333334,
      "cc_by_type": {
       "noul": 44.53333333333334
      },
      "cc_tiered": 39.86803139553988,
      "cc_tiered_open": 37.668046928916496,
      "cc_tiered_sealed": 42.068015862163264,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 44.53333333333334
     },
     "score": {
      "accuracy": 0.6053333333333333,
      "calibration": 90.67157824100035,
      "cc": 49.00847977112264,
      "cc_by_type": {
       "score": 49.00847977112264
      },
      "cc_tiered": 49.24862357009654,
      "cc_tiered_open": 49.24096119484526,
      "cc_tiered_sealed": 49.25628594534781,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 49.00847977112264
     }
    },
    "raw_on_A": {
     "calibration": 79.59030598423057,
     "capability": 67.58012336905948,
     "intelligence": 55.569940753888396
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 62.37273083755468,
    "Emphasis on Accuracy 60:20:20": 57.66488173643409,
    "Emphasis on Cost 20:20:60": 59.8778849688009,
    "Emphasis on Speed 20:60:20": 71.14547220262371,
    "Intelligence only": 51.800126951055375
   }
  },
  {
   "adapter_id": "images/ninfer-qwen3.8-flash-next/spec.json:ninfer_native.NInferNativeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Igor L. / NInfer contributors",
   "axes": {
    "calibration": 90.37232453477458,
    "cost": 71.76650263280871,
    "intelligence": 48.980301026876646,
    "speed": 89.03327048907914
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03727735037984102,
      "ece_score": 92.54452992403179,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 92.54452992403179,
      "score_before_time_limit": 92.54452992403179,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.24528301886792447,
      "brier": 0.10381961467316755,
      "ece": 0.05486291160058669,
      "n": 371,
      "score": 89.02741767988266,
      "score_before_time_limit": 89.02741767988266,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.94579945799458,
      "interval90_width": 0.40876242095754284,
      "n": 369,
      "nrps": 0.08645101367263491,
      "score": 89.5450260004093,
      "score_before_time_limit": 89.5450260004093,
      "time_limit_failed_share": 0,
      "top_ece": 0.06132423315958956
     }
    },
    "score": 90.37232453477458
   },
   "capability": 69.67631278082561,
   "class": "native-logit",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 891,334 input + 0 output tokens over 1,624 decisions = 548.850985222 tokens/decision. GPU-time cost USD 0.00873205734036537/1,000; old list-price cost USD 0.0823276477832512/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00873205734036537/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00873205734036537
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "NInfer Qwen3.8-Flash-Next mixed",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 48.827164297755125,
    "I_sealed": 49.13343775599816,
    "base": 48.980301026876646,
    "excess": -2.8796611006812776,
    "gap": -0.30627345824303376,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.22063841159643,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 60.36489462094999,
       "judge": 76.26895187870797,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 36.42788129744652,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 30.434782608695656,
       "judge": 19.999999999999996,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 39.83297318422244,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.934842929432264,
       "hard": 29.854097365310107,
       "judge": 50.77541876771969,
       "standard": 31.326121574196307
      }
     },
     "sealed|choice": {
      "cc": 68.53578282889805,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 77.38526346907102,
       "hard": 61.82126696832577,
       "judge": 68.34264432029796,
       "standard": 77.82978199285628
      }
     },
     "sealed|noul": {
      "cc": 34.31380022598546,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": -0.9708737864077666,
       "judge": 52.63157894736843,
       "standard": 60.56338028169015
      }
     },
     "sealed|score": {
      "cc": 44.55073021311098,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.34071381045655,
       "hard": 36.06656765199164,
       "judge": 48.53812828291905,
       "standard": 44.64296643196477
      }
     }
    }
   },
   "jevbench_score": 67.76131625061458,
   "key": "ninfer-qwen3.8-flash-next",
   "last_measured_on": "2026-10-02",
   "licence": "Qwen Community License 1.0 (model); Apache-2.0 (engine)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9321428571428572,
    "decisive_rate": 0.7547169811320755,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.37232453477458,
     "cost": 42.53362878447125,
     "intelligence": 48.980301026876646,
     "speed": 89.33173525631216
    },
    "board_release": "v1.6.43",
    "capability": 69.67631278082561,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      30.27504047421702,
      44.819854782587115
     ],
     "B": [
      28.319423416872812,
      43.496842238713214
     ],
     "C": [
      21.02433366265071,
      36.222113908221345
     ]
    },
    "jevbench_score": 41.96785076957165,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 51,
    "rank": 78,
    "ranks": {
     "A": 78,
     "B": 75,
     "C": 82,
     "capability": 81
    },
    "scores": {
     "A": 41.96785076957165,
     "B": 40.09260225022487,
     "C": 29.144340812202532
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3100576108321548,
     "p50_s_raw": 0.08002880541607738,
     "p95_s_adjusted": 0.37616993160918355,
     "p95_s_raw": 0.11308496580459178
    },
    "views": {
     "Balanced 33:33:33": 37.79450787350779,
     "Emphasis on Accuracy 60:20:20": 36.18545094340192,
     "Emphasis on Cost 20:20:60": 33.99296424621917,
     "Emphasis on Speed 20:60:20": 44.79615638245711,
     "Intelligence only": 34.01333411780027
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 96,
   "ranked": true,
   "ranks": {
    "A": 96,
    "B": 103,
    "C": 109,
    "capability": 103
   },
   "repo": "https://github.com/igorls/ninfer",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.76131625061458,
    "B": 62.261826042946794,
    "C": 47.05646961848235
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.31898033767938616,
    "p50_s_raw": 0.08449016883969307,
    "p95_s_adjusted": 0.39165983591228726,
    "p95_s_raw": 0.12082991795614362
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-Flash-Next, mixed NVFP4/FP8/INT4 NInfer artifact",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.49584722295648,
      90.3923314283596
     ],
     "I": [
      42.680473865516596,
      53.996834913811625
     ],
     "capability": [
      64.4871361894938,
      71.33725830271337
     ],
     "composite": null,
     "seed": 73
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.37232453477458,
     "capability": 69.67631278082561,
     "intelligence": 48.980301026876646
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7613333333333333,
      "calibration": 92.54452992403179,
      "cc": 69.47794208197946,
      "cc_by_type": {
       "choice": 69.47794208197946
      },
      "cc_tiered": 69.37821062024724,
      "cc_tiered_open": 70.22063841159643,
      "cc_tiered_sealed": 68.53578282889805,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.47794208197946
     },
     "noul": {
      "accuracy": 0.696,
      "calibration": 89.02741767988266,
      "cc": 39.19999999999999,
      "cc_by_type": {
       "noul": 39.19999999999999
      },
      "cc_tiered": 35.37084076171599,
      "cc_tiered_open": 36.42788129744652,
      "cc_tiered_sealed": 34.31380022598546,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.19999999999999
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 89.5450260004093,
      "cc": 43.98717489610087,
      "cc_by_type": {
       "score": 43.98717489610087
      },
      "cc_tiered": 42.19185169866671,
      "cc_tiered_open": 39.83297318422244,
      "cc_tiered_sealed": 44.55073021311098,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.98717489610087
     }
    },
    "raw_on_A": {
     "calibration": 86.91372207226564,
     "capability": 70.43814133335954,
     "intelligence": 53.962560594453436
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.15806534344151,
    "Emphasis on Accuracy 60:20:20": 55.52441969557024,
    "Emphasis on Cost 20:20:60": 65.32493252345064,
    "Emphasis on Speed 20:60:20": 70.51347180515715,
    "Intelligence only": 47.00286613289239
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "BlackdromeAILabs",
   "axes": {
    "calibration": 83.63708212673413,
    "cost": 76.49547138218298,
    "intelligence": 25.051442411072966,
    "speed": 93.07972984765502
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07120275435939666,
      "ece_score": 85.75944912812066,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 85.75944912812066,
      "score_before_time_limit": 85.75944912812066,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.4906666666666667,
      "brier": 0.17048542395103217,
      "ece": 0.10058815024855235,
      "n": 375,
      "score": 79.88236995028953,
      "score_before_time_limit": 79.88236995028953,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9013333333333333,
      "interval90_width": 0.5231111111111111,
      "n": 375,
      "nrps": 0.13612432038172226,
      "score": 85.26942730179223,
      "score_before_time_limit": 85.26942730179223,
      "time_limit_failed_share": 0,
      "top_ece": 0.07924356679121652
     }
    },
    "score": 83.63708212673413
   },
   "capability": 54.34426226890355,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 917,953 input + 0 output tokens over 1,485 decisions = 618.15016835 tokens/decision. GPU-time cost USD 0.00607413286302175/1,000; old list-price cost USD 0.0185445050505051/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00607413286302175/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.019615120000000003
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.006074132863021752
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Noma (Blackdrome AI Labs, Qwen3.5-4B-Base derivative)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA RTX 5090 32 GB (sm_120, driver 595.99.02, Ukraine node substituted at rental, pre-registration Amendment 2) - different hardware from the RTX PRO 6000 / H100 of the earlier runs, so Speed is as measured on this card and not comparable to the earlier rows. Python packaging: uv venv from a hashed lock at the old pins (declared deviation). Isolation: author server started inside a network namespace with loopback only (unshare -n), the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own `noma serve` with the default fast path (128-token buckets + CUDA graphs).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 29.756564202105558,
    "I_sealed": 20.346320620040373,
    "base": 25.051442411072966,
    "excess": -0.42928318829052614,
    "gap": 9.410243582065185,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 53.03396058840887,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 41.491034916640444,
       "judge": 64.40342781806197,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": -30.434782608695656,
       "judge": -30.000000000000004,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 36.23573201790782,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.562287908823535,
       "hard": 33.84744031581148,
       "judge": 44.6284633982232,
       "standard": 21.759940406169587
      }
     },
     "sealed|choice": {
      "cc": 48.83838252279794,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 67.5984752223634,
       "hard": 43.79553602405458,
       "judge": 50.6366307541626,
       "standard": 46.84665682345498
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 10.344827586206895,
       "hard": -12.727272727272732,
       "judge": -38.46153846153846,
       "standard": 4.477611940298498
      }
     },
     "sealed|score": {
      "cc": 12.200579337323186,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 26.122391760234898,
       "hard": 11.720567435668894,
       "judge": 1.0388793398912077,
       "standard": 22.942246925323882
      }
     }
    }
   },
   "jevbench_score": 13.266137669977962,
   "key": "noma",
   "last_measured_on": "2026-10-16",
   "licence": "MPL-2.0 (weights and code)",
   "listing": "ranked",
   "measured_in": "v1.6.24",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "BlackdromeAILabs/noma@fac0429688a137ce451a27ad34d05e9fd47fb624",
   "noul_decisive": {
    "acc_among_decisive": 0.8848167539267016,
    "decisive": 191,
    "decisive_rate": 0.5093333333333333,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.63708212673413,
     "cost": 61.95354260266235,
     "intelligence": 25.051442411072966,
     "speed": 93.34244428975083
    },
    "board_release": "v1.6.43",
    "capability": 54.34426226890355,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 85,
    "capability_rank": 111,
    "composite_ci95": {
     "A": [
      7.641283470998092,
      17.450802665135193
     ],
     "B": [
      6.147554589025266,
      14.778800854363228
     ],
     "C": [
      5.306446854859787,
      12.11861296189944
     ]
    },
    "jevbench_score": 12.754183690755768,
    "measured_in": "v1.6.24",
    "method": "v1.6",
    "open_board_rank": 124,
    "rank": 163,
    "ranks": {
     "A": 163,
     "B": 166,
     "C": 164,
     "capability": 166
    },
    "scores": {
     "A": 12.754183690755768,
     "B": 10.578899893750625,
     "C": 8.857072007469284
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2009935579902958,
     "p50_s_raw": 0.025496778995147906,
     "p95_s_adjusted": 0.23044825975957792,
     "p95_s_raw": 0.04022412987978896
    },
    "views": {
     "Balanced 33:33:33": 11.278487206830306,
     "Emphasis on Accuracy 60:20:20": 8.561265131134954,
     "Emphasis on Cost 20:20:60": 12.67130479105097,
     "Emphasis on Speed 20:60:20": 14.230928812591463,
     "Intelligence only": 6.2886612524076995
    }
   },
   "rank": 211,
   "ranked": true,
   "ranks": {
    "A": 211,
    "B": 211,
    "C": 211,
    "capability": 201
   },
   "repo": "https://huggingface.co/BlackdromeAILabs/noma",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 13.266137669977962,
    "B": 10.856916506619445,
    "C": 9.21259560415136
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20187975100125186,
    "p50_s_raw": 0.02593987550062593,
    "p95_s_adjusted": 0.24374418011284432,
    "p95_s_raw": 0.046872090056422164
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.45454545454545453,
       "cc": 17.41950772702694,
       "cc_by_type": {
        "choice": 25,
        "noul": 50,
        "score": -22.741476818919182
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 17.41950772702694
      },
      "calibration_gen": {
       "accuracy": 0.2871287128712871,
       "cc": -16.632615001550132,
       "cc_by_type": {
        "choice": 14.103923647932142,
        "noul": -43.99999999999999,
        "score": -20.00176865258254
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.5247524752475248,
       "cc": 7.918370014291946,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -18.518518518518523,
        "score": -25.206046235353607
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 7.918370014291946
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 55.05881191793984,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 45.17643575381953
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 55.05881191793984
      },
      "jaggedness": {
       "accuracy": 0.4388888888888889,
       "cc": -1.015847243318935,
       "cc_by_type": {
        "choice": 33.58999037536093,
        "noul": -40.54054054054054,
        "score": 3.90300843522281
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.7188498402555911,
       "cc": 52.54172250879653,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 46.39175257731958,
        "score": 46.10177807313447
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 52.54172250879653
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -29.786333031651964,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 0,
        "score": -60.326741030439756
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -27.07773849436275,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -100,
        "score": 5.364722661241627
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5163636363636364,
       "cc": 17.772590134559422,
       "cc_by_type": {
        "choice": 49.135963482230196,
        "noul": -30.555555555555557,
        "score": 34.73736247700363
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 17.772590134559422
      },
      "negation_pair_gen": {
       "accuracy": 0.4186046511627907,
       "cc": -14.28969892052313,
       "cc_by_type": {
        "choice": 43.33333333333332,
        "noul": -53.333333333333336,
        "score": -32.869096761569374
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.7708333333333334,
       "cc": 70.70572569906791,
       "cc_by_type": {
        "choice": 70.70572569906791
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 70.70572569906791
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 70.18476184672892,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 80,
        "score": 66.59032157622278
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 70.18476184672892
      },
      "usecase": {
       "accuracy": 0.4409090909090909,
       "cc": 10.091614151059419,
       "cc_by_type": {
        "choice": 44.85294117647061,
        "noul": -37.5,
        "score": 22.921901276707647
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 10.091614151059419
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -27.048739826395394,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": -50,
        "score": -16.86050519347191
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.2857142857142857,
       "cc": 47.368421052631575,
       "cc_by_type": {
        "choice": -5.263157894736845,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 47.368421052631575
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 10.925357399486423,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": -13.153267279267311
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 10.925357399486423
      },
      "de": {
       "accuracy": 0.59375,
       "cc": 30.560566461628643,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 0,
        "score": 31.504708234443456
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 30.560566461628643
      },
      "el": {
       "accuracy": 0.6,
       "cc": 44.32658194414518,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -33.333333333333336,
        "score": 99.83263223839458
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 44.32658194414518
      },
      "en": {
       "accuracy": 0.5369244135534318,
       "cc": 19.628069056743954,
       "cc_by_type": {
        "choice": 49.69551262493742,
        "noul": -5.755395683453235,
        "score": 14.944090228747676
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 19.628069056743954
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 57.38676236262902,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 75,
        "score": 16.736143531606473
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 57.38676236262902
      },
      "fi": {
       "accuracy": 0.375,
       "cc": -5.7500144985855615,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -100,
        "score": 15.536841750144958
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 37.1638514750788,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 53.84449560170699
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 37.1638514750788
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": 1.2012258532770967,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -75,
        "score": 28.883007168769836
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 1.2012258532770967
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 78.87962593695318,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 50.66849589347838
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 78.87962593695318
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 24.526026775767367,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 10.766324965904172
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 24.526026775767367
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 3.947561358412102,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 11.84268407523631
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 3.947561358412102
      },
      "mixed": {
       "accuracy": 0.6428571428571429,
       "cc": 55.142380630907915,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 48.760475226057075
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 55.142380630907915
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -2.1042632801837207,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -100,
        "score": 71.60928808152676
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5641025641025641,
       "cc": 18.96793208728958,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -60,
        "score": 39.756102652050714
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 18.96793208728958
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 21.53686440182103,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": -19.999999999999996,
        "score": 48.25231388902231
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 21.53686440182103
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": 0.8117545513000266,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60,
        "score": 64.56292322836816
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0.8117545513000266
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -8.704425914860943,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": 0,
        "score": -17.41762557066979
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 62.31405497077973,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 51.458293944597244
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 62.31405497077973
      },
      "zh": {
       "accuracy": 0.4,
       "cc": -15.528851182756002,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336,
        "score": -36.982033774256706
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6133333333333333,
       "cc": 50.0564496067663,
       "cc_by_type": {
        "choice": 50.0564496067663
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 50.0564496067663
      },
      "noul": {
       "accuracy": 0.45066666666666666,
       "cc": -9.866666666666667,
       "cc_by_type": {
        "noul": -9.866666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.464,
       "cc": 18.86027982285219,
       "cc_by_type": {
        "score": 18.86027982285219
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 18.86027982285219
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -37.645864323073745,
       "cc_by_type": {
        "choice": 20,
        "noul": -75,
        "score": -57.93759296922125
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 8.8939425965539,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -33.333333333333336,
        "score": 15.91578224100746
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 8.8939425965539
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 36.874031663183274,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": 59.40258279442787
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 36.874031663183274
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -22.785404404592622,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": 36.906944680958986
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.35294117647058826,
       "cc": -18.60576964001473,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100,
        "score": 2.153705572709419
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -32.66952881898021,
       "cc_by_type": {
        "choice": -0.62893081761006,
        "noul": -66.66666666666667,
        "score": -30.7129889726639
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 10.016228768867249,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 30.889022441055523
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 10.016228768867249
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -20.483787632035405,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100,
        "score": 11.719368811210861
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 19.43280212374197,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60,
        "score": 59.96507303789258
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 19.43280212374197
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -5.167247968174982,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": -33.333333333333336,
        "score": 7.767349600114604
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.44,
       "cc": 14.161840905735474,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": -55.55555555555556,
        "score": 21.296892226250353
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 14.161840905735474
      },
      "legal_compliance": {
       "accuracy": 0.1,
       "cc": -45.33045811238148,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -6.959116272628307
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 24.176631357497374,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": 38.59515318957466
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 24.176631357497374
      },
      "model_routing": {
       "accuracy": 0.7188940092165899,
       "cc": 49.58116367259822,
       "cc_by_type": {
        "choice": 71.94805194805195,
        "noul": 45.45454545454546,
        "score": 31.340893615197263
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 49.58116367259822
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": -5.966226657212023,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -100,
        "score": 51.19920870398773
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 44.43452582725566,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 33.396427992444785
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 44.43452582725566
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": -2.43732824862776,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -100,
        "score": 7.4663403772694314
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.30434782608695654,
       "cc": -9.883258988665188,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": -66.66666666666667,
        "score": 23.973411439801538
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 86.81002782020666,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 71.1062400442499
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 86.81002782020666
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.23201053404956,
      85.02161075963176
     ],
     "I": [
      20.510858836085088,
      28.433139202732264
     ],
     "capability": [
      49.788189765031966,
      55.97490663550985
     ],
     "composite": null,
     "seed": 56
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.63708212673413,
     "capability": 54.34426226890355,
     "intelligence": 25.051442411072966
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6133333333333333,
      "calibration": 85.75944912812066,
      "cc": 50.0564496067663,
      "cc_by_type": {
       "choice": 50.0564496067663
      },
      "cc_tiered": 50.93617155560341,
      "cc_tiered_open": 53.03396058840887,
      "cc_tiered_sealed": 48.83838252279794,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 50.0564496067663
     },
     "noul": {
      "accuracy": 0.45066666666666666,
      "calibration": 79.88236995028953,
      "cc": -9.866666666666667,
      "cc_by_type": {
       "noul": -9.866666666666667
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.464,
      "calibration": 85.26942730179223,
      "cc": 18.86027982285219,
      "cc_by_type": {
       "score": 18.86027982285219
      },
      "cc_tiered": 24.218155677615503,
      "cc_tiered_open": 36.23573201790782,
      "cc_tiered_sealed": 12.200579337323186,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 18.86027982285219
     }
    },
    "raw_on_A": {
     "calibration": 81.23569238761198,
     "capability": 52.611491162690655,
     "intelligence": 23.987289937769326
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 11.816136193214415,
    "Emphasis on Accuracy 60:20:20": 8.742438463859092,
    "Emphasis on Cost 20:20:60": 13.96482706516655,
    "Emphasis on Speed 20:60:20": 14.728178165509755,
    "Intelligence only": 6.2886612524076995
   }
  },
  {
   "adapter_id": "images/open-alternative-jev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "IkerMoel",
   "axes": {
    "calibration": 75.94384178347806,
    "cost": 63.27859776193043,
    "intelligence": 22.34610151162883,
    "speed": 90.86432704430521
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.15207997234343346,
      "ece_score": 69.58400553131331,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 69.58400553131331,
      "score_before_time_limit": 69.58400553131331,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.4,
      "brier": 0.1703546052656481,
      "ece": 0.0957881739676287,
      "n": 375,
      "score": 80.84236520647426,
      "score_before_time_limit": 80.84236520647426,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8933333333333333,
      "interval90_width": 0.5735555555555554,
      "n": 375,
      "nrps": 0.14203249622577255,
      "score": 77.40515461264664,
      "score_before_time_limit": 77.40515461264664,
      "time_limit_failed_share": 0,
      "top_ece": 0.1549322057606473
     }
    },
    "score": 75.94384178347806
   },
   "capability": 49.14497164755345,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     6.319511462595341,
     15.808691476316723
    ],
    "B": [
     5.047963685923234,
     13.390389942400779
    ],
    "C": [
     4.388549626802321,
     10.97825796966439
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (proxy tokens, I-2): exact base-model market reference; reference deepinfra:Qwen/Qwen3.5-4B frozen at the 25 Sep cut-off; the snapshot records deprecation on 11 Jun 2026 and replacement by Qwen/Qwen3.5-9B; the frozen snapshot rates remain in use (I-3)",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0167512315270936
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "open-alternative-jev (Qwen3.5-4B, IkerMoel)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 25.032294599047653,
    "I_sealed": 19.659908424210006,
    "base": 22.34610151162883,
    "excess": 2.7989985323994033,
    "gap": 5.372386174837647,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 51.43479500294909,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 64.97297297297297,
       "hard": 43.378420887071414,
       "judge": 60.44825313117996,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0.8184955141476893,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 28.57142857142858,
       "hard": 4.347826086956519,
       "judge": -19.999999999999996,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 22.843593280046196,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 40.9867728497478,
       "hard": 12.570359188934388,
       "judge": 28.72169648259133,
       "standard": 25.50131687360131
      }
     },
     "sealed|choice": {
      "cc": 44.97077692151599,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 57.43108417707487,
       "hard": 39.93212669683257,
       "judge": 40.409683426443216,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 0.6994422333695622,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 32.00000000000001,
       "hard": -14.563106796116509,
       "judge": 2.6315789473684292,
       "standard": 12.676056338028175
      }
     },
     "sealed|score": {
      "cc": 13.309506117744464,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 44.90364228457862,
       "hard": 10.653374925310422,
       "judge": 6.439637126414932,
       "standard": 13.129503906189765
      }
     }
    }
   },
   "jevbench_score": 9.429815011124056,
   "key": "open-alternative-jev",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (code and weights)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8666666666666667,
    "decisive_rate": 0.6,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 75.94384178347806,
     "cost": 63.27859776193043,
     "intelligence": 22.34610151162883,
     "speed": 91.30216983336405
    },
    "board_release": "v1.6.43",
    "capability": 49.14497164755345,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 93,
    "capability_rank": 120,
    "composite_ci95": {
     "A": [
      6.323116186163818,
      15.819609663846963
     ],
     "B": [
      5.049803290865915,
      13.396655191684498
     ],
     "C": [
      4.391052907058209,
      10.985840044338168
     ]
    },
    "jevbench_score": 9.435692571873393,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 134,
    "rank": 173,
    "ranks": {
     "A": 173,
     "B": 175,
     "C": 174,
     "capability": 178
    },
    "scores": {
     "A": 9.435692571873393,
     "B": 7.716442780206986,
     "C": 6.5525642860231885
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2366679080063477,
     "p50_s_raw": 0.04333395400317386,
     "p95_s_adjusted": 0.3130715797946323,
     "p95_s_raw": 0.08153578989731614
    },
    "views": {
     "Balanced 33:33:33": 8.37993057455771,
     "Emphasis on Accuracy 60:20:20": 6.202799597209423,
     "Emphasis on Cost 20:20:60": 9.685497012220603,
     "Emphasis on Speed 20:60:20": 10.691355012814217,
     "Intelligence only": 4.463394698403469
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 216,
   "ranked": true,
   "ranks": {
    "A": 216,
    "B": 216,
    "C": 217,
    "capability": 215
   },
   "repo": "https://github.com/ikermoel/open-alternative-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 9.429815011124056,
    "B": 7.713297441806441,
    "C": 6.548482646613928
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.25050918796332555,
    "p50_s_raw": 0.050254593981662765,
    "p95_s_adjusted": 0.3271475230052602,
    "p95_s_raw": 0.08857376150263008
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B (frozen, BF16)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      70.67388183281524,
      78.71286748398254
     ],
     "I": [
      19.129525848760927,
      27.488222507229114
     ],
     "capability": [
      45.43710032043718,
      52.3768690725618
     ],
     "composite": {
      "A": [
       6.319511462595341,
       15.808691476316723
      ],
      "B": [
       5.047963685923234,
       13.390389942400779
      ],
      "C": [
       4.388549626802321,
       10.97825796966439
      ]
     },
     "seed": 74
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 75.94384178347806,
     "capability": 49.14497164755345,
     "intelligence": 22.34610151162883
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5866666666666667,
      "calibration": 69.58400553131331,
      "cc": 47.14057008610969,
      "cc_by_type": {
       "choice": 47.14057008610969
      },
      "cc_tiered": 48.20278596223254,
      "cc_tiered_open": 51.43479500294909,
      "cc_tiered_sealed": 44.97077692151599,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 47.14057008610969
     },
     "noul": {
      "accuracy": 0.52,
      "calibration": 80.84236520647426,
      "cc": 4.0000000000000036,
      "cc_by_type": {
       "noul": 4.0000000000000036
      },
      "cc_tiered": 0.7589688737586258,
      "cc_tiered_open": 0.8184955141476893,
      "cc_tiered_sealed": 0.6994422333695622,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 4.0000000000000036
     },
     "score": {
      "accuracy": 0.4106666666666667,
      "calibration": 77.40515461264664,
      "cc": 17.046666794552323,
      "cc_by_type": {
       "score": 17.046666794552323
      },
      "cc_tiered": 18.07654969889533,
      "cc_tiered_open": 22.843593280046196,
      "cc_tiered_sealed": 13.309506117744464,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 17.046666794552323
     }
    },
    "raw_on_A": {
     "calibration": 76.45819741974654,
     "capability": 51.99334325152506,
     "intelligence": 27.528489083303576
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 8.373750135533221,
    "Emphasis on Accuracy 60:20:20": 6.200767037449664,
    "Emphasis on Cost 20:20:60": 9.680542148093952,
    "Emphasis on Speed 20:60:20": 10.673264060965625,
    "Intelligence only": 4.463394698403469
   }
  },
  {
   "adapter_id": "images/open-jev-deberta-v3-large/spec.json:warm_adapter.WarmLoaded",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Kotoba Labs",
   "axes": {
    "calibration": 67.96597988813339,
    "cost": 77.5922354035203,
    "intelligence": 2.8293414417857914,
    "speed": 67.31985953521365
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.20927949454905329,
      "ece_score": 58.14410109018935,
      "mean_tvd": null,
      "n": 682,
      "n_ece_hard": 276,
      "n_tvd": 0,
      "score": 58.14410109018935,
      "score_before_time_limit": 58.14410109018935,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.848,
      "brier": 0.2599137925955484,
      "ece": 0.20243010437488554,
      "n": 375,
      "score": 59.5139791250229,
      "score_before_time_limit": 59.5139791250229,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8945945945945946,
      "interval90_width": 0.7720720720720718,
      "n": 370,
      "nrps": 0.16620611847475783,
      "score": 86.23985944918795,
      "score_before_time_limit": 86.23985944918795,
      "time_limit_failed_share": 0,
      "top_ece": 0.05449834627074163
     }
    },
    "score": 67.96597988813339
   },
   "capability": 35.39766066495959,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.0016093461644738365,
     0.1644948890001326
    ],
    "B": [
     0.0010275646952196899,
     0.1126105605200805
    ],
    "C": [
     0.0011176015031068307,
     0.11423256180564761
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005583743842364532
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "open-jev-deberta-v3-large (local CPU)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 3.3193692126143426,
    "I_sealed": 2.3393136709572406,
    "base": 2.8293414417857914,
    "excess": -1.5933321007811418,
    "gap": 0.980055541657102,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 9.958107637843028,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 22.16216216216216,
       "hard": 1.855929537590448,
       "judge": 16.941331575477918,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -71.42857142857143,
       "hard": -73.91304347826086,
       "judge": -70,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -3.536711678360427,
       "hard": -10.998114736543751,
       "judge": 10.982108090893682,
       "standard": -6.176287708459061
      }
     },
     "sealed|choice": {
      "cc": 5.620402697655405,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 17.522725593082576,
       "hard": -1.3009049773755852,
       "judge": 7.8212290502793484,
       "standard": 10.210617071067881
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -68,
       "hard": -88.3495145631068,
       "judge": -84.21052631578947,
       "standard": -91.54929577464789
      }
     },
     "sealed|score": {
      "cc": 1.3975383152163166,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -2.206062525510788,
       "hard": -5.817775736909847,
       "judge": 9.848256453782477,
       "standard": 4.953889631982955
      }
     }
    }
   },
   "jevbench_score": 0.03235273404518239,
   "key": "open-jev-deberta-v3-large",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (model card); DeBERTa-v3 keeps its own terms",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5789473684210527,
    "decisive_rate": 0.152,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 67.96597988813339,
     "cost": 77.5922354035203,
     "intelligence": 2.8293414417857914,
     "speed": 67.62138193768119
    },
    "board_release": "v1.6.43",
    "capability": 35.39766066495959,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.0016094499426836406,
      0.16453983269325992
     ],
     "B": [
      0.0010275985408014102,
      0.11262740891592737
     ],
     "C": [
      0.0011176735713080835,
      0.11426377270365268
     ]
    },
    "jevbench_score": 0.03235814777127837,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 184,
    "rank": 228,
    "ranks": {
     "A": 228,
     "B": 228,
     "C": 228,
     "capability": 215
    },
    "scores": {
     "A": 0.03235814777127837,
     "B": 0.021367993730971636,
     "C": 0.02247093595227665
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 3.5280169868841766,
     "p50_s_raw": 1.6890084934420884,
     "p95_s_adjusted": 4.901524059008808,
     "p95_s_raw": 2.3757620295044037
    },
    "views": {
     "Balanced 33:33:33": 0.0252055189654814,
     "Emphasis on Accuracy 60:20:20": 0.014715479135725748,
     "Emphasis on Cost 20:20:60": 0.0393479951952157,
     "Emphasis on Speed 20:60:20": 0.038983856596301915,
     "Intelligence only": 0.00905974708046916
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 270,
   "ranked": true,
   "ranks": {
    "A": 270,
    "B": 270,
    "C": 270,
    "capability": 257
   },
   "repo": "https://github.com/kotoba-lang/typed-decisions",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.03235273404518239,
    "B": 0.021366104953780864,
    "C": 0.02246717642026555
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 3.520035414583981,
    "p50_s_raw": 1.6850177072919905,
    "p95_s_adjusted": 5.265832172799856,
    "p95_s_raw": 2.5579160863999277
   },
   "status": {
    "answered_ok": 1427,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "microsoft/deberta-v3-large",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      62.83136705170831,
      72.03960745264857
     ],
     "I": [
      1.0164743841847181,
      4.99302167761034
     ],
     "capability": [
      32.41699387669788,
      37.92156574534849
     ],
     "composite": {
      "A": [
       0.0016093461644738365,
       0.1644948890001326
      ],
      "B": [
       0.0010275646952196899,
       0.1126105605200805
      ],
      "C": [
       0.0011176015031068307,
       0.11423256180564761
      ]
     },
     "seed": 75
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 67.96597988813339,
     "capability": 35.39766066495959,
     "intelligence": 2.8293414417857914
    },
    "per_type": {
     "choice": {
      "accuracy": 0.272,
      "calibration": 58.14410109018935,
      "cc": 6.899197635535144,
      "cc_by_type": {
       "choice": 6.899197635535144
      },
      "cc_tiered": 7.789255167749216,
      "cc_tiered_open": 9.958107637843028,
      "cc_tiered_sealed": 5.620402697655405,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 6.899197635535144
     },
     "noul": {
      "accuracy": 0.088,
      "calibration": 59.5139791250229,
      "cc": -82.4,
      "cc_by_type": {
       "noul": -82.4
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.31733333333333336,
      "calibration": 86.23985944918795,
      "cc": -0.7784697361159942,
      "cc_by_type": {
       "score": -0.7784697361159942
      },
      "cc_tiered": 0.6987691576081583,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 1.3975383152163166,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 68.63668964823074,
     "capability": 36.14242333704565,
     "intelligence": 3.64815702586055
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.09066666666666667,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.02520113914621073,
    "Emphasis on Accuracy 60:20:20": 0.01471458332965649,
    "Emphasis on Cost 20:20:60": 0.03934159097653584,
    "Emphasis on Speed 20:60:20": 0.038965003946316364,
    "Intelligence only": 0.00905974708046916
   }
  },
  {
   "adapter_id": "images/open-jev-json-canvas-joshuasp/spec.json:joshuasp_adapter.JoshuaSPCanvasAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "JoshuaSP",
   "axes": {
    "calibration": 0,
    "cost": 70.6750337639048,
    "intelligence": 61.19221807027989,
    "speed": 85.23407766636957
   },
   "calibration": {
    "parts": {},
    "score": 0
   },
   "capability": 30.596109035139946,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 884,103 input + 0 output tokens over 1,624 decisions = 544.398399015 tokens/decision. GPU-time cost USD 0.0094950853857093/1,000; old list-price cost USD 0.0489958559113301/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0094950853857093/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009495085385709303
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Open Jev JSON Canvas (JoshuaSP)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 61.36340599482574,
    "I_sealed": 61.02103014573405,
    "base": 61.19221807027989,
    "excess": -2.2310117933465556,
    "gap": 0.34237584909168817,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 60.21617418117772,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 49.040578798364265,
       "judge": 60.44825313117996,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 72.15458937198068,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 60.00000000000001,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 51.71945443131884,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.54545454545454,
       "hard": 38.88888888888889,
       "judge": 67.79661016949152,
       "standard": 51.85185185185186
      }
     },
     "sealed|choice": {
      "cc": 68.74482965663773,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 77.38526346907102,
       "hard": 60.80316742081447,
       "judge": 71.13594040968344,
       "standard": 76.7212710924991
      }
     },
     "sealed|noul": {
      "cc": 64.47674149136002,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 45.63106796116505,
       "judge": 76.3157894736842,
       "standard": 74.64788732394365
      }
     },
     "sealed|score": {
      "cc": 49.84151928920441,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 70.92198581560282,
       "hard": 46.81372549019608,
       "judge": 50.29940119760479,
       "standard": 44.6700507614213
      }
     }
    }
   },
   "jevbench_score": 0,
   "key": "open-jev-json-canvas-joshuasp",
   "last_measured_on": "2026-10-02",
   "licence": "MIT code; Apache-2.0 DiffusionGemma weights",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8490566037735849,
    "decisive_rate": 1,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 0,
     "cost": 49.295219536461964,
     "intelligence": 61.19221807027989,
     "speed": 85.53332924416175
    },
    "board_release": "v1.6.43",
    "capability": 30.596109035139946,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 129,
    "capability_rank": 158,
    "composite_ci95": {
     "A": [
      0,
      0
     ],
     "B": [
      0,
      0
     ],
     "C": [
      0,
      0
     ]
    },
    "jevbench_score": 0,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 197,
    "rank": 241,
    "ranks": {
     "A": 241,
     "B": 241,
     "C": 241,
     "capability": 227
    },
    "scores": {
     "A": 0,
     "B": 0,
     "C": 0
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.44667022498324516,
     "p50_s_raw": 0.14833511249162257,
     "p95_s_adjusted": 0.6261524326633662,
     "p95_s_raw": 0.23807621633168308
    },
    "views": {
     "Balanced 33:33:33": 60.34909628466887,
     "Emphasis on Accuracy 60:20:20": 59.99813997005745,
     "Emphasis on Cost 20:20:60": 54.674045042588084,
     "Emphasis on Speed 20:60:20": 67.78111653907041,
     "Intelligence only": 59.47929292694184
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 282,
   "ranked": true,
   "ranks": {
    "A": 282,
    "B": 282,
    "C": 282,
    "capability": 268
   },
   "repo": "https://github.com/JoshuaSP/open-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0,
    "B": 0,
    "C": 0
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.46701108301058414,
    "p50_s_raw": 0.15850554150529206,
    "p95_s_adjusted": 0.6416010205633939,
    "p95_s_raw": 0.24580051028169692
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "label",
    "noul": "label",
    "score": "label"
   },
   "underlying": "google/diffusiongemma-26B-A4B-it BF16, one-step JSON canvas",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      0,
      0
     ],
     "I": [
      54.979058028212734,
      65.63316060204029
     ],
     "capability": [
      27.489529014106367,
      32.816580301020146
     ],
     "composite": null,
     "seed": 76
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 0,
     "capability": 30.596109035139946,
     "intelligence": 61.19221807027989
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7493333333333333,
      "calibration": null,
      "cc": 67.9433134715762,
      "cc_by_type": {
       "choice": 67.9433134715762
      },
      "cc_tiered": 64.48050191890772,
      "cc_tiered_open": 60.21617418117772,
      "cc_tiered_sealed": 68.74482965663773,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.9433134715762
     },
     "noul": {
      "accuracy": 0.84,
      "calibration": null,
      "cc": 68,
      "cc_by_type": {
       "noul": 68
      },
      "cc_tiered": 68.31566543167034,
      "cc_tiered_open": 72.15458937198068,
      "cc_tiered_sealed": 64.47674149136002,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 68
     },
     "score": {
      "accuracy": 0.624,
      "calibration": null,
      "cc": 50.6140350877193,
      "cc_by_type": {
       "score": 50.6140350877193
      },
      "cc_tiered": 50.780486860261625,
      "cc_tiered_open": 51.71945443131884,
      "cc_tiered_sealed": 49.84151928920441,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.6140350877193
     }
    },
    "raw_on_A": {
     "calibration": 0,
     "capability": 31.028102732437542,
     "intelligence": 62.056205464875084
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 71.05029305779136,
    "Emphasis on Accuracy 60:20:20": 66.74898470633056,
    "Emphasis on Cost 20:20:60": 70.89971215529596,
    "Emphasis on Speed 20:60:20": 76.11693414068185,
    "Intelligence only": 61.19221807027989
   }
  },
  {
   "adapter_id": "jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Zefan Cai",
   "axes": {
    "calibration": 83.46836347105969,
    "cost": 41.72719016000595,
    "intelligence": 52.45777412397041,
    "speed": 72.14822609431056
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09909255777071642,
      "ece_score": 80.18148844585671,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 80.18148844585671,
      "score_before_time_limit": 80.18148844585671,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.32884097035040427,
      "brier": 0.10908145798489985,
      "ece": 0.08491571210766313,
      "n": 371,
      "score": 83.01685757846738,
      "score_before_time_limit": 83.01685757846738,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.991869918699187,
      "interval90_width": 0.5313911472448059,
      "n": 369,
      "nrps": 0.07069056775911403,
      "score": 87.20674438885496,
      "score_before_time_limit": 87.20674438885496,
      "time_limit_failed_share": 0,
      "top_ece": 0.09258727223189336
     }
    },
    "score": 83.46836347105969
   },
   "capability": 67.96306879751505,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,767,510 input + 0 output tokens over 1,624 decisions = 1704.1317734 tokens/decision. GPU-time cost USD 0.0875844474931956/1,000; old list-price cost USD 0.715735344827586/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0875844474931956/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0875844474931956
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Open-Jev 27B v1.1 (Zefan Cai, LoRA + decision head on Qwen3.8-27B)",
   "endpoint_condition": "Self-hosted author loader, Transformers reference path on RTX PRO 6000 Blackwell; speed is a LOWER BOUND: flash-linear-attention and causal-conv1d kernels were not installed. The author documents the kernel path as answer-identical. Each candidate is encoded independently, producing high input-token usage.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 55.382187378509315,
    "I_sealed": 49.533360869431505,
    "base": 52.45777412397041,
    "excess": 3.275438866639566,
    "gap": 5.84882650907781,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 83.22293323536725,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 75.46398238439761,
       "judge": 88.13447593935398,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 34.03933747412009,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 65.21739130434783,
       "judge": -9.999999999999998,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 48.884291426040605,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.782717441246206,
       "hard": 50.66880107494198,
       "judge": 50.08265733375704,
       "standard": 42.568510259060396
      }
     },
     "sealed|choice": {
      "cc": 75.06513860323663,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 70.9841628959276,
       "judge": 79.51582867783985,
       "standard": 76.7212710924991
      }
     },
     "sealed|noul": {
      "cc": 22.546963950283207,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": 26.21359223300972,
       "judge": 18.421052631578938,
       "standard": 12.676056338028175
      }
     },
     "sealed|score": {
      "cc": 50.987980054774674,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 58.75384354090657,
       "hard": 44.757967203121375,
       "judge": 56.3751769445289,
       "standard": 51.48427868038395
      }
     }
    }
   },
   "jevbench_score": 40.45143841684714,
   "key": "open-jev-zefan-27b-v1.1",
   "last_measured_on": "2026-10-04",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "HF ZefanCai/Open-Jev-27B-v1.1@28cf73067d5b337860bbef3c85b8b82ba8730956; base Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0; author loader Zefan-Cai/Open-Jev@3308a15ccd7eea1df7a37d6ddc39b023b801ba16",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9437751004016064,
    "decisive_rate": 0.6711590296495957,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.46836347105969,
     "cost": 14.357426070787312,
     "intelligence": 52.45777412397041,
     "speed": 72.50871441156775
    },
    "board_release": "v1.6.43",
    "capability": 67.96306879751505,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      2.556384162487102,
      2.91320674604079
     ],
     "B": [
      2.708805295775235,
      3.1474885939684447
     ],
     "C": [
      1.7752667795049322,
      2.5853832263765106
     ]
    },
    "jevbench_score": 2.8808671915746493,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 153,
    "rank": 194,
    "ranks": {
     "A": 194,
     "B": 188,
     "C": 194,
     "capability": 92
    },
    "scores": {
     "A": 2.8808671915746493,
     "B": 3.0870580707350324,
     "C": 2.202117328789266
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.7305992921814322,
     "p50_s_raw": 0.7902996460907161,
     "p95_s_adjusted": 3.242888652905821,
     "p95_s_raw": 1.5464443264529106
    },
    "views": {
     "Balanced 33:33:33": 2.4131836334137757,
     "Emphasis on Accuracy 60:20:20": 2.9315893503528665,
     "Emphasis on Cost 20:20:60": 1.7049710123080262,
     "Emphasis on Speed 20:60:20": 3.1691813006586966,
     "Intelligence only": 4.3253676470159474
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 144,
   "ranked": true,
   "ranks": {
    "A": 144,
    "B": 140,
    "C": 136,
    "capability": 118
   },
   "repo": "https://huggingface.co/ZefanCai/Open-Jev-27B-v1.1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 40.45143841684714,
    "B": 39.60236848096451,
    "C": 30.920833064679197
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.787382136657834,
    "p50_s_raw": 0.818691068328917,
    "p95_s_adjusted": 3.411614702455699,
    "p95_s_raw": 1.6308073512278496
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.8-27B (lineage verified by architecture)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.1715005785796,
      85.37530323313025
     ],
     "I": [
      47.50165176644351,
      56.56652604734154
     ],
     "capability": [
      64.35002446863409,
      69.77751267945519
     ],
     "composite": null,
     "seed": 77
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.46836347105969,
     "capability": 67.96306879751505,
     "intelligence": 52.45777412397041
    },
    "per_type": {
     "choice": {
      "accuracy": 0.816,
      "calibration": 80.18148844585671,
      "cc": 76.46902797381657,
      "cc_by_type": {
       "choice": 76.46902797381657
      },
      "cc_tiered": 79.14403591930194,
      "cc_tiered_open": 83.22293323536725,
      "cc_tiered_sealed": 75.06513860323663,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.46902797381657
     },
     "noul": {
      "accuracy": 0.6266666666666667,
      "calibration": 83.01685757846738,
      "cc": 25.333333333333343,
      "cc_by_type": {
       "noul": 25.333333333333343
      },
      "cc_tiered": 28.293150712201648,
      "cc_tiered_open": 34.03933747412009,
      "cc_tiered_sealed": 22.546963950283207,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 25.333333333333343
     },
     "score": {
      "accuracy": 0.608,
      "calibration": 87.20674438885496,
      "cc": 50.311304174784674,
      "cc_by_type": {
       "score": 50.311304174784674
      },
      "cc_tiered": 49.936135740407636,
      "cc_tiered_open": 48.884291426040605,
      "cc_tiered_sealed": 50.987980054774674,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.311304174784674
     }
    },
    "raw_on_A": {
     "calibration": 81.23345471191158,
     "capability": 67.87352962184931,
     "intelligence": 54.51360453178707
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 36.727814207992836,
    "Emphasis on Accuracy 60:20:20": 36.6504116496222,
    "Emphasis on Cost 20:20:60": 33.22223080941593,
    "Emphasis on Speed 20:60:20": 41.1576591864662,
    "Intelligence only": 36.53491759615954
   }
  },
  {
   "adapter_id": "images/open-jev-zefan-2b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Zefan Cai (@Zefan_Cai)",
   "axes": {
    "calibration": 73.61830158128652,
    "cost": 69.61164724940987,
    "intelligence": 21.804441188912403,
    "speed": 77.13082993864059
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1271563639749637,
      "ece_score": 74.56872720500726,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 74.56872720500726,
      "score_before_time_limit": 74.56872720500726,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3261455525606469,
      "brier": 0.22536579202900195,
      "ece": 0.1725448435767295,
      "n": 371,
      "score": 65.4910312846541,
      "score_before_time_limit": 65.4910312846541,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9322493224932249,
      "interval90_width": 0.6887985546522127,
      "n": 369,
      "nrps": 0.14871270327718153,
      "score": 80.79514625419819,
      "score_before_time_limit": 80.79514625419819,
      "time_limit_failed_share": 0,
      "top_ece": 0.11769218581942746
     }
    },
    "score": 73.61830158128652
   },
   "capability": 47.71137138509946,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,767,510 input + 0 output tokens over 1,624 decisions = 1704.1317734 tokens/decision. GPU-time cost USD 0.0103025585586246/1,000; old list-price cost USD 0.170413177339901/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0103025585586246/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0103025585586246
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Open-Jev 2B (Zefan Cai)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 27.422707291556733,
    "I_sealed": 16.38819706190457,
    "base": 21.905452176730652,
    "excess": 8.46112258721392,
    "gap": 11.034510229652163,
    "penalty": 0.9953887741278608,
    "per_type_split": {
     "open|choice": {
      "cc": 41.30120233813892,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 33.837837837837846,
       "hard": 26.39194715319283,
       "judge": 44.62755438365195,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 17.126293995859218,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 30.434782608695656,
       "judge": -19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 23.84062554067206,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 35.74006952878567,
       "hard": 15.775595079225702,
       "judge": 40.5382915597717,
       "standard": 8.974465440858504
      }
     },
     "sealed|choice": {
      "cc": 35.268479371367654,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 41.46774074347795,
       "hard": 27.714932126696816,
       "judge": 38.54748603351957,
       "standard": 42.35743318142629
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 15.999999999999993,
       "hard": -28.155339805825243,
       "judge": -5.263157894736848,
       "standard": 12.676056338028175
      }
     },
     "sealed|score": {
      "cc": 13.89611181434606,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 33.49212867759059,
       "hard": -0.08856264182495188,
       "judge": 18.858024093873603,
       "standard": 24.62458387577451
      }
     }
    }
   },
   "jevbench_score": 8.766158492730982,
   "key": "open-jev-zefan-2b",
   "last_measured_on": "2026-10-02",
   "licence": "MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.748,
    "decisive_rate": 0.6738544474393531,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.61830158128652,
     "cost": 33.05490478272432,
     "intelligence": 21.804441188912403,
     "speed": 77.13731150142542
    },
    "board_release": "v1.6.43",
    "capability": 47.71137138509946,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      1.6444373649451645,
      4.563766368468016
     ],
     "B": [
      1.361709687197464,
      4.0476062027996145
     ],
     "C": [
      1.1419703923230309,
      3.169282200325012
     ]
    },
    "jevbench_score": 3.2383990318693976,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 152,
    "rank": 192,
    "ranks": {
     "A": 192,
     "B": 192,
     "C": 193,
     "capability": 180
    },
    "scores": {
     "A": 3.2383990318693976,
     "B": 2.798036205873629,
     "C": 2.248888216575971
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.0230181839782744,
     "p50_s_raw": 0.43650909198913723,
     "p95_s_adjusted": 1.8896679471712559,
     "p95_s_raw": 0.869833973585628
    },
    "views": {
     "Balanced 33:33:33": 2.799165550074912,
     "Emphasis on Accuracy 60:20:20": 2.2985053544500897,
     "Emphasis on Cost 20:20:60": 2.7782159796690187,
     "Emphasis on Speed 20:60:20": 3.6135076182350976,
     "Intelligence only": 1.812285794889051
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 219,
   "ranked": true,
   "ranks": {
    "A": 219,
    "B": 220,
    "C": 220,
    "capability": 218
   },
   "repo": "https://github.com/Zefan-Cai/Open-Jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 8.766158492730982,
    "B": 7.168868547247457,
    "C": 6.087610064396516
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.05225186990574,
    "p50_s_raw": 0.45112593495287,
    "p95_s_adjusted": 1.839913050853647,
    "p95_s_raw": 0.8449565254268235
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-2B plus rank-8 LoRA and trained scalar decision head",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.27712105156903,
      76.2730193438662
     ],
     "I": [
      16.64935744693625,
      25.12585533438709
     ],
     "capability": [
      42.961468081385505,
      49.91119595082718
     ],
     "composite": null,
     "seed": 78
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.61830158128652,
     "capability": 47.71137138509946,
     "intelligence": 21.804441188912403
    },
    "per_type": {
     "choice": {
      "accuracy": 0.496,
      "calibration": 74.56872720500726,
      "cc": 35.545598363062794,
      "cc_by_type": {
       "choice": 35.545598363062794
      },
      "cc_tiered": 38.284840854753284,
      "cc_tiered_open": 41.30120233813892,
      "cc_tiered_sealed": 35.268479371367654,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 35.545598363062794
     },
     "noul": {
      "accuracy": 0.49866666666666665,
      "calibration": 65.4910312846541,
      "cc": -0.2666666666666706,
      "cc_by_type": {
       "noul": -0.2666666666666706
      },
      "cc_tiered": 8.563146997929609,
      "cc_tiered_open": 17.126293995859218,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.456,
      "calibration": 80.79514625419819,
      "cc": 15.96828593404923,
      "cc_by_type": {
       "score": 15.96828593404923
      },
      "cc_tiered": 18.86836867750906,
      "cc_tiered_open": 23.84062554067206,
      "cc_tiered_sealed": 13.89611181434606,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 15.96828593404923
     }
    },
    "raw_on_A": {
     "calibration": 74.44730043893902,
     "capability": 49.372558434366844,
     "intelligence": 24.297816429794665
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 7.794780987253041,
    "Emphasis on Accuracy 60:20:20": 5.76573093342097,
    "Emphasis on Cost 20:20:60": 9.32923122506137,
    "Emphasis on Speed 20:60:20": 9.592842493383017,
    "Intelligence only": 4.146626072761514
   }
  },
  {
   "adapter_id": "images/open-jev-zefan-9b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Zefan Cai (@Zefan_Cai)",
   "axes": {
    "calibration": 79.80507069734225,
    "cost": 52.71772037731421,
    "intelligence": 43.88591848753404,
    "speed": 74.31878072613569
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0965793794449889,
      "ece_score": 80.68412411100222,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 80.68412411100222,
      "score_before_time_limit": 80.68412411100222,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.18867924528301883,
      "brier": 0.1465943912444352,
      "ece": 0.11119868678055905,
      "n": 371,
      "score": 77.76026264388818,
      "score_before_time_limit": 77.76026264388818,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.926829268292683,
      "interval90_width": 0.46047877145438126,
      "n": 369,
      "nrps": 0.10981645578089093,
      "score": 80.97082533713633,
      "score_before_time_limit": 80.97082533713633,
      "time_limit_failed_share": 0,
      "top_ece": 0.13538351873819116
     }
    },
    "score": 79.80507069734225
   },
   "capability": 61.84549459243814,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,767,510 input + 0 output tokens over 1,624 decisions = 1704.1317734 tokens/decision. GPU-time cost USD 0.0376769715675016/1,000; old list-price cost USD 0.170413177339901/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0376769715675016/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03767697156750158
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Open-Jev 9B (Zefan Cai)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 45.51928930503784,
    "I_sealed": 42.25254767003025,
    "base": 43.88591848753404,
    "excess": 0.6933539925693402,
    "gap": 3.266741635007584,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 61.95574336697866,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 64.1396665618119,
       "judge": 56.49307844429796,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 35.9399585921325,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 47.82608695652173,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 38.66216595600236,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 45.958227241534885,
       "hard": 49.02462444279136,
       "judge": 29.24271270673138,
       "standard": 28.41839821356458
      }
     },
     "sealed|choice": {
      "cc": 56.97677837170926,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 64.08247727440693,
       "hard": 48.076923076923066,
       "judge": 59.96275605214154,
       "standard": 66.7446729892844
      }
     },
     "sealed|noul": {
      "cc": 36.79892764867179,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 48,
       "hard": 20.388349514563096,
       "judge": 44.736842105263165,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 32.98193698970971,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 51.23313360297058,
       "hard": 27.657644820832182,
       "judge": 32.23017056028181,
       "standard": 35.63257266497618
      }
     }
    }
   },
   "jevbench_score": 45.49003604451487,
   "key": "open-jev-zefan-9b",
   "last_measured_on": "2026-10-02",
   "licence": "MIT (loader); Apache-2.0 (adapter and pinned Qwen base); CC0-1.0 public training projection",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8637873754152824,
    "decisive_rate": 0.8113207547169812,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.80507069734225,
     "cost": 33.05490478272432,
     "intelligence": 43.88591848753404,
     "speed": 74.39486349899005
    },
    "board_release": "v1.6.43",
    "capability": 61.84549459243814,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      12.01117001715265,
      21.4767289933671
     ],
     "B": [
      11.375313778606227,
      21.19925799700656
     ],
     "C": [
      8.341090289689344,
      14.91439513428271
     ]
    },
    "jevbench_score": 17.045611746249723,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 111,
    "rank": 147,
    "ranks": {
     "A": 147,
     "B": 143,
     "C": 149,
     "capability": 121
    },
    "scores": {
     "A": 17.045611746249723,
     "B": 16.53766290698564,
     "C": 11.837230379340085
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.3850112770218401,
     "p50_s_raw": 0.6175056385109201,
     "p95_s_adjusted": 2.624583175848238,
     "p95_s_raw": 1.237291587924119
    },
    "views": {
     "Balanced 33:33:33": 15.193819551111247,
     "Emphasis on Accuracy 60:20:20": 15.024035626101695,
     "Emphasis on Cost 20:20:60": 13.257314615761418,
     "Emphasis on Speed 20:60:20": 18.03145694648765,
     "Intelligence only": 14.776357163022537
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 129,
   "ranked": true,
   "ranks": {
    "A": 129,
    "B": 130,
    "C": 134,
    "capability": 151
   },
   "repo": "https://github.com/Zefan-Cai/Open-Jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 45.49003604451487,
    "B": 42.54991567274617,
    "C": 31.590302808690883
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.4373612958937882,
    "p50_s_raw": 0.6436806479468942,
    "p95_s_adjusted": 2.5736884943442417,
    "p95_s_raw": 1.211844247172121
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-9B plus rank-8 LoRA and trained scalar decision head",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      73.96736762912185,
      81.45578416794778
     ],
     "I": [
      37.82741036517579,
      48.601933428170696
     ],
     "capability": [
      56.87009089863343,
      64.27945845500723
     ],
     "composite": null,
     "seed": 79
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.80507069734225,
     "capability": 61.84549459243814,
     "intelligence": 43.88591848753404
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6733333333333333,
      "calibration": 80.68412411100222,
      "cc": 58.22399893902218,
      "cc_by_type": {
       "choice": 58.22399893902218
      },
      "cc_tiered": 59.466260869343955,
      "cc_tiered_open": 61.95574336697866,
      "cc_tiered_sealed": 56.97677837170926,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 58.22399893902218
     },
     "noul": {
      "accuracy": 0.6933333333333334,
      "calibration": 77.76026264388818,
      "cc": 38.66666666666667,
      "cc_by_type": {
       "noul": 38.66666666666667
      },
      "cc_tiered": 36.369443120402146,
      "cc_tiered_open": 35.9399585921325,
      "cc_tiered_sealed": 36.79892764867179,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 38.66666666666667
     },
     "score": {
      "accuracy": 0.536,
      "calibration": 80.97082533713633,
      "cc": 35.27556488047482,
      "cc_by_type": {
       "score": 35.27556488047482
      },
      "cc_tiered": 35.82205147285603,
      "cc_tiered_open": 38.66216595600236,
      "cc_tiered_sealed": 32.98193698970971,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.27556488047482
     }
    },
    "raw_on_A": {
     "calibration": 76.09059647397694,
     "capability": 61.58634472209667,
     "intelligence": 47.0820929702164
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 41.86076460263527,
    "Emphasis on Accuracy 60:20:20": 38.220000295813406,
    "Emphasis on Cost 20:20:60": 41.3526446726818,
    "Emphasis on Speed 20:60:20": 46.90521268765749,
    "Intelligence only": 33.80925240677986
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 598,
    "round": "score-a5-1",
    "subset": "A5"
   },
   "api_flag": true,
   "author": "OpenAI",
   "axes": {
    "calibration": 90.1,
    "cost": 48.6,
    "intelligence": 56.9,
    "speed": 88.31
   },
   "calibration": {
    "score": 90.1
   },
   "capability": 73.5,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     51.52,
     64.44
    ],
    "B": [
     48.52,
     63.63
    ],
    "C": [
     35.78,
     64.44
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A5 u P run x tariff USD 0.1/M input, 0.0/M output (spec openai-decisions.json; OpenAI Decisions API (POST /v1/decisions, gpt-6-luna) list price USD 0.10 per 1M input tokens, no output/cache charges; source https://developers.openai.com/api/docs/guides/decisions (checked 2026-10-06); actual usage), per 1,000 decisions on the v1.6.1 common cost basis (592 of 600 items; the 8 A5 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a5.json). List price as of 6 Oct 2026 (launch day); re-checked at each revision.",
    "kind": "tariff",
    "usd_per_1000": 0.0517
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "OpenAI Decisions (gpt-6-luna)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 58.07,
    "I_sealed": 44.16,
    "base": 56.9,
    "per_type_split": {
     "open|choice": {
      "cc": 72.5513,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.1081,
       "hard": 62.2523,
       "judge": 80.2241,
       "standard": 69.8619
      }
     },
     "open|noul": {
      "cc": 48.9124,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.1429,
       "hard": 65.2174,
       "judge": 20,
       "standard": 55.5556
      }
     },
     "open|score": {
      "cc": 52.7419,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.2727,
       "hard": 53.7315,
       "judge": 54.4068,
       "standard": 50
      }
     },
     "sealed|choice": {
      "cc": 69.2318,
      "n": {
       "easy": 17,
       "hard": 73,
       "judge": 38,
       "standard": 22
      },
      "tiers": {
       "easy": 75.8389,
       "hard": 64.8643,
       "judge": 76.453,
       "standard": 63.8312
      }
     },
     "sealed|noul": {
      "cc": 20.6086,
      "n": {
       "easy": 15,
       "hard": 17,
       "judge": 22,
       "standard": 21
      },
      "tiers": {
       "easy": 33.3333,
       "hard": 17.6471,
       "judge": 18.1818,
       "standard": 23.8095
      }
     },
     "sealed|score": {
      "cc": 42.6453,
      "n": {
       "easy": 9,
       "hard": 30,
       "judge": 21,
       "standard": 15
      },
      "tiers": {
       "easy": 55.64,
       "hard": 50.4536,
       "judge": 35.5753,
       "standard": 31.1364
      }
     }
    }
   },
   "jevbench_score": 62.37,
   "key": "openai-decisions",
   "last_measured_on": "2026-10-06",
   "licence": "hosted API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 22,
    "axes": {
     "calibration": 90.1,
     "cost": 48.6,
     "intelligence": 56.9,
     "speed": 89.25
    },
    "board_release": "v1.6.43",
    "capability": 73.5,
    "capability_api_board_rank": 15,
    "capability_open_board_rank": null,
    "capability_rank": 29,
    "composite_ci95": {
     "A": [
      51.61,
      64.57
     ],
     "B": [
      48.59,
      63.73
     ],
     "C": [
      35.84,
      64.57
     ]
    },
    "jevbench_score": 62.5,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 53,
    "ranks": {
     "A": 53,
     "B": 51,
     "C": 46,
     "capability": 53
    },
    "scores": {
     "A": 62.5,
     "B": 60.53,
     "C": 56.2
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 598,
     "p50_s_adjusted": 0.298,
     "p50_s_raw": 0.298,
     "p95_s_adjusted": 0.399,
     "p95_s_raw": 0.399
    },
    "views": null
   },
   "rank": 106,
   "ranked": true,
   "ranks": {
    "A": 106,
    "B": 107,
    "C": 90,
    "capability": 71
   },
   "repo": null,
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 62.37,
    "B": 60.43,
    "C": 56.09
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.305,
    "p50_s_raw": 0.305,
    "p95_s_adjusted": 0.484,
    "p95_s_raw": 0.484
   },
   "status": {
    "answered_ok": 598,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A5",
    "equated": true,
    "equating_offset": {
     "C": 6.152420143549691,
     "I": 5.780609498995467
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a5-1",
    "run_sha256": "83c37a9258466d201a3a3fb30f5245bd2fa77f29f3c02b8594e68934a7bb77aa"
   }
  },
  {
   "adapter_id": "images/opendecision/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Deepan Wadhwa",
   "axes": {
    "calibration": 68.56606448002526,
    "cost": 79.10704862374335,
    "intelligence": 7.425225288997898,
    "speed": 86.36099014084385
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.18936051642522217,
      "ece_score": 62.12789671495557,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 62.12789671495557,
      "score_before_time_limit": 62.12789671495557,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.4933333333333333,
      "brier": 0.2702030712841029,
      "ece": 0.17767917616168663,
      "n": 375,
      "score": 64.46416476766268,
      "score_before_time_limit": 64.46416476766268,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9706666666666667,
      "interval90_width": 0.8675555555555556,
      "n": 375,
      "nrps": 0.1718997562232394,
      "score": 79.10613195745756,
      "score_before_time_limit": 79.10613195745756,
      "time_limit_failed_share": 0,
      "top_ece": 0.12298880231380463
     }
    },
    "score": 68.56606448002526
   },
   "capability": 37.99564488451158,
   "class": "classifier",
   "composite_ci95": {
    "A": [
     0.20915427982595058,
     1.1140594117388836
    ],
    "B": [
     0.14360793060164476,
     0.8080395435414361
    ],
    "C": [
     0.1452460276569101,
     0.773652369263114
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004970862068965518
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "OpenDecision (ModernBERT-large zero-shot)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 8.199956224645156,
    "I_sealed": 6.6504943533506395,
    "base": 7.425225288997898,
    "excess": -1.0239257711437273,
    "gap": 1.5494618712945165,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 21.530744131863024,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 37.72972972972973,
       "hard": -3.8062283737024116,
       "judge": 32.76203032300594,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -14.28571428571429,
       "hard": -82.6086956521739,
       "judge": -30.000000000000004,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 3.069124542072446,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 29.089033648823246,
       "hard": -4.629529940171362,
       "judge": 10.925488522812966,
       "standard": -6.32806701792612
      }
     },
     "sealed|choice": {
      "cc": 18.786220529284186,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 34.8163476461459,
       "hard": 9.898190045248851,
       "judge": 19.92551210428307,
       "standard": 26.838280576425678
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 12.00000000000001,
       "hard": -39.80582524271845,
       "judge": -50,
       "standard": -15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 1.165262530767735,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 23.523223159689433,
       "hard": -5.374879041668379,
       "judge": 2.0574839819155533,
       "standard": 1.728233184457384
      }
     }
    }
   },
   "jevbench_score": 0.5084959848229078,
   "key": "opendecision",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6842105263157895,
    "decisive_rate": 0.5066666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 68.56606448002526,
     "cost": 79.10704862374335,
     "intelligence": 7.425225288997898,
     "speed": 87.12856375997052
    },
    "board_release": "v1.6.43",
    "capability": 37.99564488451158,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 117,
    "capability_rank": 145,
    "composite_ci95": {
     "A": [
      0.20924922812147978,
      1.114873836219044
     ],
     "B": [
      0.14364373295734356,
      0.808381046176735
     ],
     "C": [
      0.14531196397324977,
      0.7742179418187807
     ]
    },
    "jevbench_score": 0.508795165212109,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 162,
    "rank": 205,
    "ranks": {
     "A": 205,
     "B": 206,
     "C": 205,
     "capability": 209
    },
    "scores": {
     "A": 0.508795165212109,
     "B": 0.3579489209247008,
     "C": 0.3533299758417423
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.29369962783530357,
     "p50_s_raw": 0.07184981391765177,
     "p95_s_adjusted": 0.6595386148197576,
     "p95_s_raw": 0.2547693074098788
    },
    "views": {
     "Balanced 33:33:33": 0.41664409033845246,
     "Emphasis on Accuracy 60:20:20": 0.25754719918644864,
     "Emphasis on Cost 20:20:60": 0.5990327904113044,
     "Emphasis on Speed 20:60:20": 0.6067044473606604,
     "Intelligence only": 0.1637528610901445
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 247,
   "ranked": true,
   "ranks": {
    "A": 247,
    "B": 248,
    "C": 247,
    "capability": 251
   },
   "repo": "https://github.com/deepanwadhwa/OpenDecision",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.5084959848229078,
    "B": 0.3578304284057583,
    "C": 0.3531222116825748
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.335404502786696,
    "p50_s_raw": 0.09270225139334798,
    "p95_s_adjusted": 0.6891790978377684,
    "p95_s_raw": 0.2695895489188842
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "MoritzLaurer/ModernBERT-large-zeroshot-v2.0 (~400M) used as a zero-shot NLI decision engine",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      62.90521170003539,
      71.97774479185684
     ],
     "I": [
      5.421041180341186,
      9.877186440823028
     ],
     "capability": [
      34.738025189088674,
      40.24525966167053
     ],
     "composite": {
      "A": [
       0.20915427982595058,
       1.1140594117388836
      ],
      "B": [
       0.14360793060164476,
       0.8080395435414361
      ],
      "C": [
       0.1452460276569101,
       0.773652369263114
      ]
     },
     "seed": 80
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 68.56606448002526,
     "capability": 37.99564488451158,
     "intelligence": 7.425225288997898
    },
    "per_type": {
     "choice": {
      "accuracy": 0.36933333333333335,
      "calibration": 62.12789671495557,
      "cc": 19.346740808806086,
      "cc_by_type": {
       "choice": 19.346740808806086
      },
      "cc_tiered": 20.158482330573605,
      "cc_tiered_open": 21.530744131863024,
      "cc_tiered_sealed": 18.786220529284186,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 19.346740808806086
     },
     "noul": {
      "accuracy": 0.3466666666666667,
      "calibration": 64.46416476766268,
      "cc": -30.666666666666664,
      "cc_by_type": {
       "noul": -30.666666666666664
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.3253333333333333,
      "calibration": 79.10613195745756,
      "cc": 2.4048951483054193,
      "cc_by_type": {
       "score": 2.4048951483054193
      },
      "cc_tiered": 2.1171935364200904,
      "cc_tiered_open": 3.069124542072446,
      "cc_tiered_sealed": 1.165262530767735,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 2.4048951483054193
     }
    },
    "raw_on_A": {
     "calibration": 66.1740117306899,
     "capability": 37.492707526259565,
     "intelligence": 8.811403321829234
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.41637660935819926,
    "Emphasis on Accuracy 60:20:20": 0.2574858508477883,
    "Emphasis on Cost 20:20:60": 0.5987010081282494,
    "Emphasis on Speed 20:60:20": 0.605684594386601,
    "Intelligence only": 0.1637528610901445
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "ekzhang",
   "axes": {
    "calibration": 78.57,
    "cost": 35.63,
    "intelligence": 38.77,
    "speed": 77.66
   },
   "calibration": {
    "score": 78.57
   },
   "capability": 58.67,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     8.74,
     26.17
    ],
    "B": [
     7.93,
     25.59
    ],
    "C": [
     6.07,
     18.17
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)",
    "kind": "estimate",
    "usd_per_1000": 0.13981028325123152
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "openjev-sglang (Qwen3.6-35B-A3B on SGLang)",
   "endpoint_kind": "demo",
   "intelligence": {
    "I_open": 44.34,
    "I_sealed": 28.06,
    "base": 38.77,
    "per_type_split": {
     "open|choice": {
      "cc": 69.7112,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.3243,
       "hard": 50.928,
       "judge": 68.3586,
       "standard": 100
      }
     },
     "open|noul": {
      "cc": 32.4458,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 39.1304,
       "judge": 20,
       "standard": 11.1111
      }
     },
     "open|score": {
      "cc": 30.8736,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 41.725,
       "hard": 32.6506,
       "judge": 31.5762,
       "standard": 20.8401
      }
     },
     "sealed|choice": {
      "cc": 56.672,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 77.7572,
       "hard": 37.9511,
       "judge": 63.8191,
       "standard": 72.8507
      }
     },
     "sealed|noul": {
      "cc": 2.9813,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 41.1765,
       "hard": -9.0909,
       "judge": 0,
       "standard": 12.5
      }
     },
     "sealed|score": {
      "cc": 24.5338,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 60.2254,
       "hard": 0.9111,
       "judge": 15.1583,
       "standard": 67.9965
      }
     }
    }
   },
   "jevbench_score": 15.38,
   "key": "openjev-sglang",
   "last_measured_on": "2026-10-06",
   "licence": "no licence file in the repository as of 2026-09-19; Qwen3.6 weights keep their own terms",
   "listing": "ranked",
   "method": "v1.7",
   "open": "yes",
   "previous": {
    "api_board_rank": 39,
    "axes": {
     "calibration": 78.57,
     "cost": 35.63,
     "intelligence": 38.77,
     "speed": 77.71
    },
    "board_release": "v1.6.43",
    "capability": 58.67,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      8.74,
      26.17
     ],
     "B": [
      7.94,
      25.59
     ],
     "C": [
      6.07,
      18.18
     ]
    },
    "jevbench_score": 15.38,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 160,
    "ranks": {
     "A": 160,
     "B": 151,
     "C": 161,
     "capability": 139
    },
    "scores": {
     "A": 15.38,
     "B": 14.51,
     "C": 10.68
    },
    "speed": {
     "adjustment": "x2 (demo)",
     "n": 591,
     "p50_s_adjusted": 1.265,
     "p50_s_raw": 0.6325,
     "p95_s_adjusted": 1.339,
     "p95_s_raw": 0.6695
    },
    "views": null
   },
   "rank": 205,
   "ranked": true,
   "ranks": {
    "A": 205,
    "B": 196,
    "C": 205,
    "capability": 172
   },
   "repo": "https://github.com/ekzhang/openjev-sglang",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 15.38,
    "B": 14.51,
    "C": 10.68
   },
   "speed": {
    "adjustment": "x2 (demo)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.27,
    "p50_s_raw": 0.635,
    "p95_s_adjusted": 1.35,
    "p95_s_raw": 0.675
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "16b2211d50e366983ff0094cc9cd19139fb0bdee7907d67f0240d1f03f900217"
   }
  },
  {
   "adapter_id": "images/openjev-thinking/spec.json:openjev_thinking.ThinkingOpenJev",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "razorback16",
   "axes": {
    "calibration": 81.33711087001693,
    "cost": 54.86348744521688,
    "intelligence": 70.21463489688583,
    "speed": 72.91370874195911
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1238179439505489,
      "ece_score": 75.23641120989022,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 75.23641120989022,
      "score_before_time_limit": 75.23641120989022,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.04312668463611857,
      "brier": 0.06619006264403647,
      "ece": 0.07056572946447708,
      "n": 371,
      "score": 85.6578224961523,
      "score_before_time_limit": 85.88685410710458,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.8455284552845529,
      "interval90_width": 0.06526648599819332,
      "n": 369,
      "nrps": 0.05970047185852581,
      "score": 83.11709890400827,
      "score_before_time_limit": 83.11709890400827,
      "time_limit_failed_share": 0,
      "top_ece": 0.13897877503065434
     }
    },
    "score": 81.33711087001693
   },
   "capability": 75.77587288345137,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,383,393 input + 592,069 output tokens over 1,624 decisions = 1832.18103448 tokens/decision. GPU-time cost USD 0.0319558532794633/1,000; old list-price cost USD 0.241456939655172/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0319558532794633/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03195585327946334
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "OpenJev (thinking, BF16)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 68.15800897274833,
    "I_sealed": 72.2712608210233,
    "base": 70.21463489688583,
    "excess": -6.686639490713214,
    "gap": -4.11325184827497,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 71.32537528109535,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 67.9144385026738,
       "judge": 72.31377719182596,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 68.51345755693583,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 80,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 64.63519408021384,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 75.65890690670581,
       "hard": 83.20743211500854,
       "judge": 51.76582051786083,
       "standard": 41.282921940907926
      }
     },
     "sealed|choice": {
      "cc": 72.81145995047186,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 86.6972138053359,
       "hard": 66.4027149321267,
       "judge": 74.86033519553072,
       "standard": 75.61276019214189
      }
     },
     "sealed|noul": {
      "cc": 75.64956278293162,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 61.165048543689316,
       "judge": 81.57894736842107,
       "standard": 91.54929577464787
      }
     },
     "sealed|score": {
      "cc": 68.35275972966642,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 85.29967322252621,
       "hard": 58.44270568724672,
       "judge": 72.99731091086852,
       "standard": 72.73258429627272
      }
     }
    }
   },
   "jevbench_score": 68.40125787286172,
   "key": "openjev-thinking",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9322033898305084,
    "decisive_rate": 0.9567567567567568,
    "supported": true,
    "valid": 370
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.41345474033434,
     "cost": 28.51480924396573,
     "intelligence": 70.34621384425424,
     "speed": 72.55270827201397
    },
    "board_release": "v1.6.43",
    "capability": 75.87983429229429,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      16.909474945094924,
      17.502684272940023
     ],
     "B": [
      17.624044058024868,
      18.54119755452704
     ],
     "C": [
      16.909474945094924,
      17.502684272940023
     ]
    },
    "jevbench_score": 17.265218966202397,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 110,
    "rank": 146,
    "ranks": {
     "A": 146,
     "B": 136,
     "C": 126,
     "capability": 41
    },
    "scores": {
     "A": 17.265218966202397,
     "B": 18.156241294771178,
     "C": 17.265218966202397
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.8639664588030427,
     "p50_s_raw": 0.8569832294015214,
     "p95_s_adjusted": 2.980513406405225,
     "p95_s_raw": 1.4152567032026127
    },
    "views": {
     "Balanced 33:33:33": 15.470816250462342,
     "Emphasis on Accuracy 60:20:20": 17.77278752746646,
     "Emphasis on Cost 20:20:60": 12.207984243040777,
     "Emphasis on Speed 20:60:20": 17.94233922790431,
     "Intelligence only": 22.879243501891544
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 92,
   "ranked": true,
   "ranks": {
    "A": 92,
    "B": 63,
    "C": 65,
    "capability": 59
   },
   "repo": "https://github.com/razorback16/openjev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.40125787286172,
    "B": 68.75640115114027,
    "C": 68.40125787286172
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.9280300891026854,
    "p50_s_raw": 0.8890150445513427,
    "p95_s_adjusted": 2.651644620718434,
    "p95_s_raw": 1.250822310359217
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/diffusiongemma-26b-a4b-it, BF16; OpenJev think=512",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.19017694535097,
      83.80603684194338
     ],
     "I": [
      64.92265020833241,
      74.56158589207546
     ],
     "capability": [
      71.75831354064398,
      78.68433165919848
     ],
     "composite": null,
     "seed": 81
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.33711087001693,
     "capability": 75.77587288345137,
     "intelligence": 70.21463489688583
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7946666666666666,
      "calibration": 75.23641120989022,
      "cc": 73.74079933309964,
      "cc_by_type": {
       "choice": 73.74079933309964
      },
      "cc_tiered": 72.0684176157836,
      "cc_tiered_open": 71.32537528109535,
      "cc_tiered_sealed": 72.81145995047186,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.74079933309964
     },
     "noul": {
      "accuracy": 0.88,
      "calibration": 85.6578224961523,
      "cc": 76,
      "cc_by_type": {
       "noul": 76
      },
      "cc_tiered": 72.08151016993372,
      "cc_tiered_open": 68.51345755693583,
      "cc_tiered_sealed": 75.64956278293162,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 76
     },
     "score": {
      "accuracy": 0.7786666666666666,
      "calibration": 83.11709890400827,
      "cc": 68.45852348469666,
      "cc_by_type": {
       "score": 68.45852348469666
      },
      "cc_tiered": 66.49397690494013,
      "cc_tiered_open": 64.63519408021384,
      "cc_tiered_sealed": 68.35275972966642,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.45852348469666
     }
    },
    "raw_on_A": {
     "calibration": 79.80009682824573,
     "capability": 76.44304334504619,
     "intelligence": 73.08598986184666
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.95763729156265,
    "Emphasis on Accuracy 60:20:20": 66.96305757955713,
    "Emphasis on Cost 20:20:60": 60.504806380044954,
    "Emphasis on Speed 20:60:20": 67.92220413927541,
    "Intelligence only": 70.21463489688583
   }
  },
  {
   "adapter_id": "images/openjev-verdict/spec.json:warm_adapter.WarmLoaded",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Hemant (heman10x)",
   "axes": {
    "calibration": 49.11476555745841,
    "cost": 86.62313527343974,
    "intelligence": 4.1496399816416485,
    "speed": 78.45185691662326
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.3268574231249395,
      "ece_score": 34.6285153750121,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 34.6285153750121,
      "score_before_time_limit": 34.6285153750121,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.6666666666666667,
      "brier": 0.28308617598601715,
      "ece": 0.20554105202175244,
      "n": 375,
      "score": 58.89178959564951,
      "score_before_time_limit": 58.89178959564951,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.816,
      "interval90_width": 0.6600000000000001,
      "n": 375,
      "nrps": 0.21771137902642015,
      "score": 53.823991701713624,
      "score_before_time_limit": 53.823991701713624,
      "time_limit_failed_share": 0,
      "top_ece": 0.3529043934696537
     }
    },
    "score": 49.11476555745841
   },
   "capability": 26.63220276955003,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.016675550357464684,
     0.2745367164429247
    ],
    "B": [
     0.010945992361712387,
     0.1924055928378838
    ],
    "C": [
     0.011580243303794922,
     0.1906504975298088
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.002791871921182266
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "openJev Verdict (heman10x, ModernBERT-base 151M)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 5.710480975731317,
    "I_sealed": 2.5887989875519795,
    "base": 4.1496399816416485,
    "excess": 0.5482943457410934,
    "gap": 3.121681988179337,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 17.13144292719395,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": 13.180245360176167,
       "judge": 20.896506262359924,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -56.52173913043479,
       "judge": -80,
       "standard": -55.55555555555556
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 8.056046691320095,
       "hard": -27.078088834020786,
       "judge": -9.574770356665475,
       "standard": -54.57566450814868
      }
     },
     "sealed|choice": {
      "cc": 7.766396962655938,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 10.87133249575052,
       "hard": 5.316742081447947,
       "judge": 6.890130353817524,
       "standard": 12.427638871782255
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -64,
       "hard": -49.51456310679612,
       "judge": -63.1578947368421,
       "standard": -60.56338028169015
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 11.069847893733552,
       "hard": -5.166740337464448,
       "judge": -15.492495207027046,
       "standard": -16.23629733590537
      }
     }
    }
   },
   "jevbench_score": 0.09645563919820467,
   "key": "openjev-verdict",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6,
    "decisive_rate": 0.3333333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 49.11476555745841,
     "cost": 86.62313527343974,
     "intelligence": 4.1496399816416485,
     "speed": 79.40046783928
    },
    "board_release": "v1.6.43",
    "capability": 26.63220276955003,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 131,
    "capability_rank": 161,
    "composite_ci95": {
     "A": [
      0.016680739292552642,
      0.27473431836523593
     ],
     "B": [
      0.010947780723182459,
      0.1924832136378699
     ],
     "C": [
      0.011583846730939337,
      0.19078772108696943
     ]
    },
    "jevbench_score": 0.09650709186186587,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 177,
    "rank": 221,
    "ranks": {
     "A": 221,
     "B": 221,
     "C": 221,
     "capability": 232
    },
    "scores": {
     "A": 0.09650709186186587,
     "B": 0.06541514759096348,
     "C": 0.0670188137929624
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.6885442301630974,
     "p50_s_raw": 0.2692721150815487,
     "p95_s_adjusted": 1.66732926396653,
     "p95_s_raw": 0.7586646319832651
    },
    "views": {
     "Balanced 33:33:33": 0.07793885393113759,
     "Emphasis on Accuracy 60:20:20": 0.046097376471726674,
     "Emphasis on Cost 20:20:60": 0.11949201576637343,
     "Emphasis on Speed 20:60:20": 0.11862755697857134,
     "Intelligence only": 0.028581910146043114
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 263,
   "ranked": true,
   "ranks": {
    "A": 263,
    "B": 263,
    "C": 263,
    "capability": 273
   },
   "repo": "https://github.com/Heman10x-NGU/openJev-verdict-2.0",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.09645563919820467,
    "B": 0.06539623103617571,
    "C": 0.066983082776531
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.7912336127832532,
    "p50_s_raw": 0.3206168063916266,
    "p95_s_adjusted": 1.8051345552317797,
    "p95_s_raw": 0.8275672776158899
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "GLiClass ModernBERT-base (knowledgator/gliclass-modern-base-v2.0) fine-tuned, 151M",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      43.627592951745086,
      53.02973228229396
     ],
     "I": [
      2.2588266004870072,
      6.027430783705419
     ],
     "capability": [
      23.293166910490978,
      29.139107914545924
     ],
     "composite": {
      "A": [
       0.016675550357464684,
       0.2745367164429247
      ],
      "B": [
       0.010945992361712387,
       0.1924055928378838
      ],
      "C": [
       0.011580243303794922,
       0.1906504975298088
      ]
     },
     "seed": 82
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 49.11476555745841,
     "capability": 26.63220276955003,
     "intelligence": 4.1496399816416485
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2906666666666667,
      "calibration": 34.6285153750121,
      "cc": 9.286397696162448,
      "cc_by_type": {
       "choice": 9.286397696162448
      },
      "cc_tiered": 12.448919944924945,
      "cc_tiered_open": 17.13144292719395,
      "cc_tiered_sealed": 7.766396962655938,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 9.286397696162448
     },
     "noul": {
      "accuracy": 0.2,
      "calibration": 58.89178959564951,
      "cc": -60,
      "cc_by_type": {
       "noul": -60
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2693333333333333,
      "calibration": 53.823991701713624,
      "cc": -9.675453706526316,
      "cc_by_type": {
       "score": -9.675453706526316
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 49.678974901443674,
     "capability": 27.759290067036318,
     "intelligence": 5.839605232628962
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.07789411162231974,
    "Emphasis on Accuracy 60:20:20": 0.04608798194516172,
    "Emphasis on Cost 20:20:60": 0.11942891140749813,
    "Emphasis on Speed 20:60:20": 0.1184411678748766,
    "Intelligence only": 0.028581910146043114
   }
  },
  {
   "adapter_id": "images/openjev-verdict-1.4/spec.json:warm_adapter.WarmLoaded",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Hemant (heman10x)",
   "axes": {
    "calibration": 75.69890703057408,
    "cost": 86.62313527343974,
    "intelligence": 5.035901218168153,
    "speed": 80.93559275082234
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13155601347182505,
      "ece_score": 73.68879730563499,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 73.68879730563499,
      "score_before_time_limit": 73.68879730563499,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 1,
      "brier": 0.2495580281877627,
      "ece": 0.12226520797197521,
      "n": 375,
      "score": 75.54695840560495,
      "score_before_time_limit": 75.54695840560495,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9893333333333333,
      "interval90_width": 0.916888888888889,
      "n": 375,
      "nrps": 0.17500942351683016,
      "score": 77.8609653804823,
      "score_before_time_limit": 77.8609653804823,
      "time_limit_failed_share": 0,
      "top_ece": 0.1338856344367619
     }
    },
    "score": 75.69890703057408
   },
   "capability": 40.36740412437111,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.037381561171560676,
     0.4670303161269418
    ],
    "B": [
     0.02459567039766373,
     0.3261942564122147
    ],
    "C": [
     0.02595941748025047,
     0.32432660842148725
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.002791871921182266
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "openJev Verdict 1.4",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 5.214507036728864,
    "I_sealed": 4.857295399607441,
    "base": 5.035901218168153,
    "excess": -2.216176005316821,
    "gap": 0.3572116371214227,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 15.643521110186592,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -5.081081081081076,
       "hard": 13.180245360176167,
       "judge": 24.851680949241928,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 11.206824837702234,
       "hard": -8.9038551649125,
       "judge": -0.9107640863542654,
       "standard": -33.62275309182012
      }
     },
     "sealed|choice": {
      "cc": 8.257811980618104,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 5.550218017884884,
       "hard": 6.334841628959255,
       "judge": 9.683426443202995,
       "standard": 11.319127971425068
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "sealed|score": {
      "cc": 6.314074218204219,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 14.960976046364117,
       "hard": 5.800113224444569,
       "judge": 5.914716157590583,
       "standard": 3.617582382564022
      }
     }
    }
   },
   "jevbench_score": 0.17216463673473972,
   "key": "openjev-verdict-1.4",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": null,
    "decisive_rate": 0,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 75.69890703057408,
     "cost": 86.62313527343974,
     "intelligence": 5.035901218168153,
     "speed": 81.49274954261848
    },
    "board_release": "v1.6.43",
    "capability": 40.36740412437111,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 113,
    "capability_rank": 141,
    "composite_ci95": {
     "A": [
      0.0373899767734964,
      0.4672540094953807
     ],
     "B": [
      0.024598584677982253,
      0.3262815362283655
     ],
     "C": [
      0.025965261648261392,
      0.3244819510384588
     ]
    },
    "jevbench_score": 0.17222636557331997,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 172,
    "rank": 216,
    "ranks": {
     "A": 216,
     "B": 217,
     "C": 216,
     "capability": 205
    },
    "scores": {
     "A": 0.17222636557331997,
     "B": 0.11682093815444984,
     "C": 0.11960164275925
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.7352308319881559,
     "p50_s_raw": 0.2926154159940779,
     "p95_s_adjusted": 0.9644980074837803,
     "p95_s_raw": 0.40724900374189016
    },
    "views": {
     "Balanced 33:33:33": 0.13684263027905905,
     "Emphasis on Accuracy 60:20:20": 0.08186844689007573,
     "Emphasis on Cost 20:20:60": 0.2066197471065469,
     "Emphasis on Speed 20:60:20": 0.20540349835976665,
     "Intelligence only": 0.05108478843903597
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 257,
   "ranked": true,
   "ranks": {
    "A": 257,
    "B": 258,
    "C": 257,
    "capability": 247
   },
   "repo": "https://huggingface.co/heman10x/rlcd-modernbert-151m",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.17216463673473972,
    "B": 0.11679821380829648,
    "C": 0.11955877551023596
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.8207729119807482,
    "p50_s_raw": 0.3353864559903741,
    "p95_s_adjusted": 0.9822402124293148,
    "p95_s_raw": 0.4161201062146574
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "GLiClass ModernBERT-base fine-tuned decision model, 151M; v1.4 fixed inference engine",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      70.20079626876355,
      79.44394588474857
     ],
     "I": [
      2.9611688986465445,
      7.176652677169861
     ],
     "capability": [
      37.175911743687855,
      42.54492005899604
     ],
     "composite": {
      "A": [
       0.037381561171560676,
       0.4670303161269418
      ],
      "B": [
       0.02459567039766373,
       0.3261942564122147
      ],
      "C": [
       0.02595941748025047,
       0.32432660842148725
      ]
     },
     "seed": 83
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 75.69890703057408,
     "capability": 40.36740412437111,
     "intelligence": 5.035901218168153
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2866666666666667,
      "calibration": 73.68879730563499,
      "cc": 8.774854826028024,
      "cc_by_type": {
       "choice": 8.774854826028024
      },
      "cc_tiered": 11.950666545402349,
      "cc_tiered_open": 15.643521110186592,
      "cc_tiered_sealed": 8.257811980618104,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 8.774854826028024
     },
     "noul": {
      "accuracy": 0,
      "calibration": 75.54695840560495,
      "cc": -100,
      "cc_by_type": {
       "noul": -100
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.344,
      "calibration": 77.8609653804823,
      "cc": 4.055284730748376,
      "cc_by_type": {
       "score": 4.055284730748376
      },
      "cc_tiered": 3.1570371091021094,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 6.314074218204219,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 4.055284730748376
     }
    },
    "raw_on_A": {
     "calibration": 73.35601823970862,
     "capability": 39.682189077168275,
     "intelligence": 6.008359914627925
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.13679067123541433,
    "Emphasis on Accuracy 60:20:20": 0.08185728575415545,
    "Emphasis on Cost 20:20:60": 0.2065486701646049,
    "Emphasis on Speed 20:60:20": 0.20519291407630152,
    "Intelligence only": 0.05108478843903597
   }
  },
  {
   "adapter_id": "images/opensourcejev-qwen35-4b-q4km/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "sabeel111",
   "axes": {
    "calibration": 73.92484912653177,
    "cost": 83.8256538982382,
    "intelligence": 23.217995107845883,
    "speed": 75.84507183307446
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.20911848208469058,
      "ece_score": 58.17630358306188,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 58.17630358306188,
      "score_before_time_limit": 58.17630358306188,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.5121293800539084,
      "brier": 0.1552006734049543,
      "ece": 0.11625450134770884,
      "n": 371,
      "score": 76.74909973045823,
      "score_before_time_limit": 76.74909973045823,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9566395663956639,
      "interval90_width": 0.5600722673893404,
      "n": 369,
      "nrps": 0.11625319455925476,
      "score": 86.84914406607521,
      "score_before_time_limit": 86.84914406607521,
      "time_limit_failed_share": 0,
      "top_ece": 0.07338196205962055
     }
    },
    "score": 73.92484912653177
   },
   "capability": 48.571422117188824,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 570,303 input + 1,624 output tokens over 1,624 decisions = 352.17179803 tokens/decision. GPU-time cost USD 0.00346054794023613/1,000; old list-price cost USD 0.0106851539408867/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00346054794023613/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003460547940236125
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "OpenSourceJev (Qwen3.5-4B Q4_K_M, native llama.cpp)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 24.81038701116274,
    "I_sealed": 21.62560320452903,
    "base": 23.217995107845883,
    "excess": 0.6113961641954666,
    "gap": 3.1847838066337104,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 41.32946193915506,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 24.50456118276188,
       "judge": 64.40342781806196,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -4.347826086956519,
       "judge": -30.000000000000004,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 33.101699094333156,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.34023030303031,
       "hard": 17.52584074074074,
       "judge": 47.55760169491523,
       "standard": 25.450296296296294
      }
     },
     "sealed|choice": {
      "cc": 35.66176229147756,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 48.11913384081,
       "hard": 23.133484162895915,
       "judge": 42.271880819366864,
       "standard": 44.57445498214066
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 24,
       "hard": -30.097087378640776,
       "judge": -7.8947368421052655,
       "standard": -4.225352112676061
      }
     },
     "sealed|score": {
      "cc": 29.21504732210953,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 58.979268794326224,
       "hard": 18.989292892156882,
       "judge": 31.412349700598806,
       "standard": 31.488491878172564
      }
     }
    }
   },
   "jevbench_score": 10.555669166289977,
   "key": "opensourcejev-qwen35-4b-q4km",
   "last_measured_on": "2026-10-02",
   "licence": "MIT (repository code); Apache-2.0 (Qwen/Qwen3.5-4B base and unsloth/Qwen3.5-4B-GGUF Q4_K_M conversion)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9502762430939227,
    "decisive_rate": 0.48787061994609165,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.92484912653177,
     "cost": 69.13657649738097,
     "intelligence": 23.217995107845883,
     "speed": 77.06362388237805
    },
    "board_release": "v1.6.43",
    "capability": 48.571422117188824,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 94,
    "capability_rank": 121,
    "composite_ci95": {
     "A": [
      6.483328619758367,
      14.579365175086792
     ],
     "B": [
      5.195391797769789,
      12.338152010991331
     ],
     "C": [
      4.502311541498866,
      10.124559149365826
     ]
    },
    "jevbench_score": 10.263493713193125,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 132,
    "rank": 171,
    "ranks": {
     "A": 171,
     "B": 171,
     "C": 171,
     "capability": 179
    },
    "scores": {
     "A": 10.263493713193125,
     "B": 8.482178412964995,
     "C": 7.127426189717446
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.7395113246515393,
     "p50_s_raw": 0.29475566232576966,
     "p95_s_adjusted": 2.6588435446377843,
     "p95_s_raw": 1.2544217723188922
    },
    "views": {
     "Balanced 33:33:33": 9.174389111825883,
     "Emphasis on Accuracy 60:20:20": 6.882520870049428,
     "Emphasis on Cost 20:20:60": 10.842363781630336,
     "Emphasis on Speed 20:60:20": 11.176826098501683,
     "Intelligence only": 5.0064990418048145
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 214,
   "ranked": true,
   "ranks": {
    "A": 214,
    "B": 214,
    "C": 214,
    "capability": 217
   },
   "repo": "https://github.com/sabeel111/OpenSourceJev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 10.555669166289977,
    "B": 8.640299177679745,
    "C": 7.330325809923594
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.7913347106426954,
    "p50_s_raw": 0.3206673553213477,
    "p95_s_adjusted": 3.2895205278880892,
    "p95_s_raw": 1.5697602639440447
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.41904601995815,
      77.70943106496745
     ],
     "I": [
      19.379013533491335,
      26.692735209928347
     ],
     "capability": [
      44.976218992193104,
      51.58836189201757
     ],
     "composite": null,
     "seed": 84
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.92484912653177,
     "capability": 48.571422117188824,
     "intelligence": 23.217995107845883
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5093333333333333,
      "calibration": 58.17630358306188,
      "cc": 37.25074126351086,
      "cc_by_type": {
       "choice": 37.25074126351086
      },
      "cc_tiered": 38.495612115316305,
      "cc_tiered_open": 41.32946193915506,
      "cc_tiered_sealed": 35.66176229147756,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 37.25074126351086
     },
     "noul": {
      "accuracy": 0.45866666666666667,
      "calibration": 76.74909973045823,
      "cc": -8.266666666666666,
      "cc_by_type": {
       "noul": -8.266666666666666
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.544,
      "calibration": 86.84914406607521,
      "cc": 30.80778824561413,
      "cc_by_type": {
       "score": 30.80778824561413
      },
      "cc_tiered": 31.15837320822134,
      "cc_tiered_open": 33.101699094333156,
      "cc_tiered_sealed": 29.21504732210953,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.80778824561413
     }
    },
    "raw_on_A": {
     "calibration": 73.9791977813484,
     "capability": 49.500913389632025,
     "intelligence": 25.022628997915646
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 9.48737495390115,
    "Emphasis on Accuracy 60:20:20": 6.986260582494583,
    "Emphasis on Cost 20:20:60": 11.713505038761635,
    "Emphasis on Speed 20:60:20": 11.402499759823042,
    "Intelligence only": 5.0064990418048145
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "PhillyLabs (Hugging Face)",
   "axes": {
    "calibration": 82.5161952575238,
    "cost": 79.09341092589379,
    "intelligence": 56.08237887792767,
    "speed": 93.1956789542462
   },
   "base_model": "Qwen/Qwen3.5-4B",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07552443779343157,
      "ece_score": 84.89511244131369,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 84.89511244131369,
      "score_before_time_limit": 84.89511244131369,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.14690235603494103,
      "ece": 0.0918769811672612,
      "n": 375,
      "score": 81.62460376654776,
      "score_before_time_limit": 81.62460376654776,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8793565683646113,
      "interval90_width": 0.38873994638069703,
      "n": 373,
      "nrps": 0.11508143690363018,
      "score": 81.02886956471,
      "score_before_time_limit": 81.02886956471,
      "time_limit_failed_share": 0,
      "top_ece": 0.13217058590108488
     }
    },
    "score": 82.5161952575238
   },
   "capability": 69.29928706772574,
   "code_revision": "THE AUTHOR'S OWN SERVING PATH, run unmodified: the repository's serve.sh (sha256 1777363ab5f1accf32137bfe71531e2b9b4fa65a234ac4d54d2dd70be9862a5a, byte-identical to the serve.sh of the live H2O-Lightning-4B v1.1 row) started from the read-only model directory with every override variable unset (MODEL, VLLM_PORT, PORT, MAX_LEN, GPU_UTIL, VLLM_ARGS and every SHIM_* knob), so the script's own defaults applied: vllm serve /model --served-model-name h2oai/h2o-lightning-4b --host 127.0.0.1 --port 8000 --max-model-len 40960 --gpu-memory-utilization 0.90, and python3 /model/h2o_lightning_shim.py --config /model/serve_config.json --vllm http://127.0.0.1:8000 --port 8741. The shim's native endpoint POST /v1/systemone was used with no response shim of ours and no chat-template or instruction conversion; the request path is the frozen v1.6 measurement driver with the frozen TypeSafe adapter (the model string is ignored by the server). Author code: h2o_lightning_shim.py (sha256 8af95357658a7258b233a3f8bcea2cb48b0297a145ee2e1fd3957006e3ffcb92, H2O's v1.1 shim with marked [v37] lines), h2o_lightning_shim_v37.py (sha256 b779d907b2a1438a4fad2ced36944368e2c1073dca72d9cd0318f9755140e98e, 368 lines, imports only json, math and pathlib) and serve_config.json (sha256 fc26d64ff050121c159fc0570cfee49560e3e7f3d3df076aa96b944c1aa4a8e3). Stack: the official image vllm/vllm-openai:v0.30.0 pinned by digest sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 (the stack of the live H2O-Lightning-4B v1.1 row; no overlay, no pip, no source checkout), with vLLM 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, tokenizers 0.23.2, triton 3.7.1 and huggingface-hub 1.32.0 asserted on the node before any weights step, and a first-party Triton compiler gate passed before the weights step and again inside the scored container. Source review (text only; no author file was imported or executed on the evaluator host; line by line over the request path and the delta against H2O's reviewed v1.1 shim): verdict PASS WITH CONDITIONS, no stop condition; no branch on item text, ids, gold labels or item lists; the delta is a small pure-Python readout patch (no I/O, network, subprocess, eval, exec or pickle); no auto_map and no trust_remote_code; weights and tokenizer are safetensors and tokenizer files only. The conditions were binding: author code only on a disposable rented pod, every override variable unset, the model directory exactly the 17 pinned files (no subdirectory, symlink or cache) checked on the node and again inside the scored container, weights fetched on the pod by first-party code at the full revision and sha256-verified, and the served configuration read back from /health. /health was compared with the preregistered configuration by strict dictionary equality in the pre-sealed smoke, in the pre-warm, right after the scored run and after the record-only replay (all equal; the same values each time). The server ran in a read-only, network-none, cap-drop-ALL, no-new-privileges container with no credentials, read-only model and input mounts and private output, after a network-cut proof (only loopback, no default route, DNS and TCP failed) and an offline model-hash proof; the scored process was a fresh server process in a fresh container whose first request was the frozen driver's own public pre-warm. Model: https://huggingface.co/PhillyLabs/philly-decider-v2 at 36e5ddde2c59021d37ed5974aa7c9f16062050e0 (17 files; model.safetensors 8,411,558,400 bytes, sha256 70941015cc8d822bfba82134dd9b86e79e722df4c1be9bc39a8c99755322138e; every file hashed against the pinned list on the node, 17 of 17 verified, and again inside the scored container, 17 of 17), about 4.2 B parameters, BF16, Qwen3_5ForCausalLM, text only, one CUDA device, no quantisation. Card base_model h2oai/h2o-lightning-4b (H2O-Lightning-4B v1.1) at 193ad740925b176a3b70a5a13a7cff2f2fadd01e; the serve_config.json inherited from H2O names Qwen/Qwen3.5-4B (851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a) as the backbone. Author code and weights were fetched by our code at the pinned revisions and ran only on the disposable rented pod (standing rule: author code only on a disposable rented pod, no infrastructure credentials, egress cut, USD and TTL limits, deletion receipt); one first self-hosted use for this row. Kit SHAS sha256 prefix 78487d76 (full 78487d76366e685be9668916f07eeceb2090e687802dd9456a996425fc1cb46a); preregistration sha256 prefix d0534e97; sealed offline token-count file sha256 df9887809a9f53393aaec208d19ee39007eedf0d82bd9717e46a2cf8c6b109f8.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 752,008 input + 0 output tokens over 1,485 decisions = 506.402693603 tokens/decision. GPU-time cost USD 0.00497606795343036/1,000; old list-price cost USD 0.0151920808080808/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00497606795343036/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.024684640000000008
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004976067953430363
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of PhillyLabs/philly-decider-v2 in this intake (submitted by the author through a public GitHub issue). FINE-TUNE OF A FINE-TUNE: the card says Philly Decider v2 is a fine-tune of H2O-Lightning-4B v1.1 by H2O.ai (h2oai/h2o-lightning-4b at 193ad740925b176a3b70a5a13a7cff2f2fadd01e, Apache-2.0, itself a post-train of Qwen/Qwen3.5-4B). The author's change is a merged LoRA delta (round-1 adapter plus 0.5 times a round-2b adapter, concatenated rank 256, merged in fp32 with one bf16 rounding) plus changed readout settings; the tokenizer, chat template, prompt format, label tokens, serve.sh and the shim's prompt and trim code are H2O's. The name is 'Jev-compatible' only: it is a Philly Decider model, not a Jev model; the author is PhillyLabs and the base model's author is H2O.ai. Under the published treatment, a disclosed fine-tune of another entrant's open model is included like any fine-tune, with the lineage named on the row. RELATED ROWS: the live H2O-Lightning-4B v1.1 row is a different row and is not replaced; nothing is inferred between the two. The same author's separate philadelphia-decider-4b is a different model; this row says nothing about it. SIBLING CHECK: the model.safetensors LFS sha256 70941015cc8d822bfba82134dd9b86e79e722df4c1be9bc39a8c99755322138e is NEW: it differs from the declared base revision (c65443b1d28891ffc4b6f7d90028c542d6ccb24c2f9cd6a46a8294a44ab0ffae), from all 50 H2O-Lightning-4B revisions, from the author's philadelphia-decider-4b and its base Mapika/decider-4b, and matches no LFS blob of the 173 live JevBench row repositories (171 compared; 2 repositories answered 401 and could not be compared: Uniphore/uniphore-4b-decider and nemotron-deciffusion/Nemotron-Deciffusion-8B-v2). The tokenizer.json is identical to the base's (record only). THE AUTHOR'S OWN SERVING PATH on our rented hardware, run unmodified (see code_revision), with no shim of ours. PROMPT (H2O v1.1's, unchanged): Qwen chat template with thinking off, the system line 'You are a decision engine...', then record, question and options with one label per option (255 labels available), compact JSON for a structured state, and an 'Answer:' prefill; one forward pass per decision, label logprobs only, max_tokens 1, no generated text, no ensemble, no retry. NATIVE OUTPUTS AND READOUT POST-PROCESSING (the author's [v37] readout, served as shipped): choice = softmax((logit + b) / 0.798298) with the position bias b = [0, 0.15, 0.2, -0.15, -0.05, 0, 0, 0] on option positions 1 to 8; noul = softmax(logit / 0.933033) and then H2O's 0.801 floor (a yes/no probability whose larger side is below 0.801 is moved to 0.801 on its own side; the answer never changes; the same device as the live H2O-Lightning-4B v1.1 row, the Calibration axis includes its effect, and we did not measure a variant without it); score = a 'mirror' readout: p = softmax(logit / 0.7), m = the median level of p, served probabilities = (p + p reflected about m) / 2, so the served score is the median level m and does NOT preserve the softmax mean. The configured temperature_by_type.score value 0.5 is never used by the mirror readout (dead setting, disclosed). READOUT FIT, AUTHOR STATEMENT (not independently verified): the author states that these per-type temperatures, the choice position bias and the mirror score readout were fitted on 887 unpublished local calibration rows ('EIKOS-CAL + REALDEV-CAL') and that no JevBench items were used. No rows, fit scripts or fit receipts are published (the repository's MERGE_MANIFEST records the weights lineage only and lists an older serve_config.json hash). Our public-only check found no alignment between the position bias and the public items' answer-position prior (Pearson r 0.00); that is weak evidence only, and the temperatures cannot be checked offline. CONFIGURATION: the shipped settings exactly as in code_revision and /health; no other setting was measured and nothing is inferred about one. TEACHERS (author statement): DeepSeek V4.1-Flash (API, thinking on; target teacher, under the DeepSeek API terms), Qwen3.8-27B-FP8 (labels on round-1 error items), and two models that are themselves JevBench entrants: Chon-1.0-12B-Fast (logit teacher where no DeepSeek label exists) and Torchcast-4B v1.2 (a yes/no second vote only, never a target). TRAINING DATA AND CARD NUMBERS (author statements, not independently verified): public decision-style datasets under permissive licences; the card says no JevBench item text and no answers were used for training or model selection and that training rows passed an 8-gram overlap filter against the public items. We did not inspect training examples or verify overlap with JevBench public or sealed items, so contamination of the sealed draw is unchecked. The card's 61.96 versus 57.92 (H2O-Lightning-4B v1.1, stock readout) are measured on the author's own 1,200-item held-out set with the author's scorer; they are not JevBench numbers and are not used here. CONTEXT AND TRIM, SILENT: a state over 32,000 tokens is cut to its first 16,000 tokens, the text '\\n...\\n' and its last 16,000 tokens before the prompt is built, with no signal in the response; the author's default and the same as the live H2O-Lightning-4B v1.1 row. The 15 trimmed items were counted offline and frozen by the evaluator before launch as 'trimmed, answered' (count only; expected-outcomes sha256 c2ccee9a2f210a8206808540a5fb52b8ec99c0dfc2eb0b24918018ee012afeb2; counts sha256 df9887809a9f53393aaec208d19ee39007eedf0d82bd9717e46a2cf8c6b109f8; preregistered) and the measured trim record equals the preregistered set exactly: 15 of 15 trimmed, no other item trimmed, no refusal, no unexpected failure. Their answers rest on the first and last 16,000 tokens of the state, not the whole state. The longest original state in the draw is 80,370 tokens; the longest untrimmed prompt is 1,856 tokens and the most options on any item is 9 (limit 255). A prompt of 40,960 tokens or more, or more than 255 options, would be refused with HTTP 422; none occurred (boundary proven on the pod before any sealed item: 40,959 tokens answered, 40,960 and 42,000 refused, 256 options refused, with recovery after each). ANSWERED: 1,500 of 1,500, HTTP 200 on every row, no refusal, no error, no native non-200 status and no deviation record. DISTRIBUTION VALIDITY: two score-type responses contain a probability 2.220446049250313e-16 above 1 from floating-point rounding. The builder accepts this within its 1e-12 validation tolerance, but does not alter the raw probabilities. The unchanged frozen scorer rejects both distributions under its strict [0,1] check: they remain invalid answers in the 375-item Score competence denominator, with the invalid-answer worst-error treatment, and are excluded from Score calibration (373 valid distributions). The supplied scores already include this treatment; the responses were not clipped, renormalised or rescored as valid. USAGE: the server-native input usage (1,234,232 tokens over all 1,500 rows, 752,008 on the common 1,485) equals our first-party count exactly, 0 mismatches; output_tokens 0 natively; the unit is the final prompt, so for the 15 trimmed items it is the trimmed prompt (the original states are not billed). HARDWARE AND LATENCY: NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB with stock vLLM 0.30.0, so latency is not comparable with the author's own figures on other hardware. 10 s RULE: 0 decisions took 10 s or longer, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); measured loopback p95 0.05 s, maximum 1.09 s over all 1,500 requests; scored loop 59 s. The pre-sealed cold first request, a trimmed prompt built from the longest original state, took 1.39 s. The v1.7 share view uses the common self-hosted latency estimate (2 times measured p50 plus 0.15 s); this adjustment is an assumption, not a measured hosted endpoint. DETERMINISM (record-only): the first 50 requests were run twice after the scored loop, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. In the pre-sealed smoke, the same body sent twice gave identical answers for all three types; two groups of unique synthetic states that differed only by a tag text gave different probabilities (record-only, not a repeat test). OFFLINE COUNTER: the same counter that produced the sealed counts was run on the public items before the launch (231 answered, 0 trimmed, 150,981 input tokens, at most 6 options) and reproduces byte for byte. PREREGISTRATION AMENDMENT 1 WAS NOT APPEND-ONLY (preregistration disclosure): it was written before any pod and before any sealed item reached a model (it used only the aggregate longest-state length from our offline sealed token count), closed the findings of an independent review, and says that it supersedes the earlier wording, so the preregistration text was edited in place rather than only appended to. It rewrote the section on the served readout (the mirror readout is now described as setting the score mean to the median level), the cold-probe line (the first POST is a cold trimmed probe at the longest original state, 80,000 changed to 80,370 tokens) and stop rule 8 (a non-200 status returned by the author's server is now passed unchanged to the frozen driver and its own unchanged stop rule instead of stopping the run). Its other fixes: the /health contract value for the readout's median spread (the author's code snaps the default 0.1 to a 2^-20 grid and publishes 0.10000038146972656), a check of the scored process's configuration at completion, and the bound on the cold probe. The counter, rendering, tokenizer pins and the sealed counts were unchanged and not re-counted. The evaluator recorded and accepted the edit and asked that it be disclosed on the row. A later amendment (RTX PRO 6000 FIN-03 only, A6000 removed after another row's pre-sealed TTL-fit stop) was a pure append to the preregistration; the evaluator confirmed the final kit hash (78487d76) again. The run did not trigger any of the amended stop rules: no native non-200 status, no deviation record and no configuration difference occurred. LICENCE NOTE: weights Apache-2.0 per the model card and the repository's LICENSE file (H2O.ai's licence file kept unchanged; NOTICE lists the author's modifications), ungated; the base h2oai/h2o-lightning-4b is Apache-2.0 and the backbone Qwen/Qwen3.5-4B is Apache-2.0; teacher labels from DeepSeek V4.1-Flash fall under the DeepSeek API terms; the disposition is for the evaluator. ATTEMPT HISTORY: one measured attempt. One earlier attempt stopped before any reservation or create call (stock race), so no instance existed and no sealed item reached a model before the measured attempt.",
   "display": "Philly Decider v2 (PhillyLabs)",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-26",
    "release": "v1.7.10",
    "results_sha256": "dd7bafcf7783111d8ab95852e5a9716f88093473df4a710d7d563bec377fa139"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB, driver 580.178.04), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, image 24.04.cuda13.0, USD 2.583095890410959/h recorded by the runner, 150 GB disk. One VM for this row, one measured attempt. Measured 26 Oct 2026, VM window about 19:32-19:43Z (639.6 s from create to closed), about USD 0.46 GPU rental (USD 0.459 at the recorded hourly rate). Author server (stock vLLM at 127.0.0.1:8000 behind the author's shim at 127.0.0.1:8741) in a --network none --read-only container, network cut before sealed input (cut proof and offline model-hash proof receipts: only loopback, no default route, DNS and TCP failed; all 17 model files verified), read-only model, source and input, private output, HF offline flags, no credentials. Before sealed input: the image digest, the stack versions, the Triton gate, the weights fetch with a node re-verify (17 of 17), the pre-sealed smoke in its own server process (/health equal to the preregistered configuration, a cold trimmed probe at the longest original state, anchors from 83 to 40,959 tokens, over-limit and 256-option refusals with recovery, typed probes) and the TTL fit gate all passed (forecast of items over 10 s: 0; forecast compute 674 s); a public pre-warm preceded the scored loop, which ran in a fresh server process. Input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b (v1.6-regular-20261012) was re-checked inside the container (match). Complete 1,500-row run, rc 0; scored loop 59 s. Raw output sha256 6d02e677e68cd68383a789e4f720e025b4c14845a02c831a2ba95f8b6f7eaf67 (the pull-hash receipt). Node-shred and local-shred receipts verified; exact-id teardown receipt confirms the VM closed. GPU rental is separate from the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 63.67847340330138,
    "I_sealed": 48.48628435255395,
    "base": 56.08237887792767,
    "excess": 5.352662280391719,
    "gap": 15.19218905074743,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 72.53391464649734,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 71.6892104435357,
       "judge": 80.22412656558998,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 68.99171842650105,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 70,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 49.50978713690577,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.54545454545454,
       "hard": 53.70370370370371,
       "judge": 40.677966101694906,
       "standard": 51.85185185185182
      }
     },
     "sealed|choice": {
      "cc": 56.006086365898184,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 77.12833545108006,
       "hard": 47.958829651902384,
       "judge": 55.04407443682666,
       "standard": 62.982493144906144
      }
     },
     "sealed|noul": {
      "cc": 53.592030203455835,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 50.90909090909091,
       "judge": 41.53846153846155,
       "standard": 64.17910447761194
      }
     },
     "sealed|score": {
      "cc": 35.860736488307836,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 57.89473684210527,
       "hard": 34.92957746478875,
       "judge": 27.60416666666664,
       "standard": 39.090909090909065
      }
     }
    }
   },
   "jevbench_score": 75.01434382924442,
   "key": "phillylabs_philly_decider_v2",
   "last_measured_on": "2026-10-26",
   "licence": "Apache-2.0 per the model card and the repository's LICENSE file (H2O.ai's licence file, kept unchanged); base h2oai/h2o-lightning-4b Apache-2.0; backbone Qwen/Qwen3.5-4B Apache-2.0; ungated",
   "listing": "ranked",
   "measured_in": "v1.7.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "PhillyLabs/philly-decider-v2@36e5ddde2c59021d37ed5974aa7c9f16062050e0 + base h2oai/h2o-lightning-4b@193ad740925b176a3b70a5a13a7cff2f2fadd01e",
   "new_in": "v1.7.10",
   "noul_decisive": {
    "acc_among_decisive": 0.8,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 19,
   "ranked": true,
   "ranks": {
    "A": 19,
    "B": 51,
    "C": 72,
    "capability": 106
   },
   "repo": "https://huggingface.co/PhillyLabs/philly-decider-v2",
   "scores": {
    "A": 75.01434382924442,
    "B": 70.2700710342455,
    "C": 65.53822395383537
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19765678199999,
    "p50_s_raw": 0.02382839099999501,
    "p95_s_adjusted": 0.24239317319999146,
    "p95_s_raw": 0.04619658659999573
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 50,
       "cc_by_type": {
        "choice": 50,
        "noul": 100,
        "score": 0
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 50
      },
      "calibration_gen": {
       "accuracy": 0.5643564356435643,
       "cc": 43.68898020501945,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": 76,
        "score": 21.875
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 43.68898020501945
      },
      "composite_gen": {
       "accuracy": 0.6534653465346535,
       "cc": 46.42484886387325,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 33.33333333333333,
        "score": 38.46153846153846
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 46.42484886387325
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 88.8888888888889,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 66.66666666666667
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 88.8888888888889
      },
      "jaggedness": {
       "accuracy": 0.5055555555555555,
       "cc": 31.56408970823284,
       "cc_by_type": {
        "choice": 33.58999037536093,
        "noul": 29.729729729729737,
        "score": 31.372549019607842
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 31.56408970823284
      },
      "llm_router": {
       "accuracy": 0.8083067092651757,
       "cc": 72.74783279277023,
       "cc_by_type": {
        "choice": 68.3014880690323,
        "noul": 85.56701030927836,
        "score": 64.375
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 72.74783279277023
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": 15.591397849462366,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": 0,
        "score": 50
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 15.591397849462366
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -19.49762722958599,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -50,
        "score": -9.523809523809511
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.7090909090909091,
       "cc": 55.76342827946679,
       "cc_by_type": {
        "choice": 63.808281708509945,
        "noul": 52.77777777777777,
        "score": 50.70422535211265
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 55.76342827946679
      },
      "negation_pair_gen": {
       "accuracy": 0.7209302325581395,
       "cc": 34.444444444444436,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 39.99999999999999,
        "score": -19.999999999999996
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 34.444444444444436
      },
      "option_order": {
       "accuracy": 0.8333333333333334,
       "cc": 78.69507323568575,
       "cc_by_type": {
        "choice": 78.69507323568575
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 78.69507323568575
      },
      "perturb_pair": {
       "accuracy": 0.8125,
       "cc": 78.17047817047816,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100,
        "score": 61.53846153846152
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 78.17047817047816
      },
      "usecase": {
       "accuracy": 0.6181818181818182,
       "cc": 49.164052287581704,
       "cc_by_type": {
        "choice": 59.55882352941178,
        "noul": 58.33333333333333,
        "score": 29.599999999999994
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 49.164052287581704
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -7.387057387057387,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50,
        "score": -76.92307692307692
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 57.77777777777777,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 39.99999999999999
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 57.77777777777777
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 69.69667820956778,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 75,
        "score": 73.91304347826086
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 69.69667820956778
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.49348230912477,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 100
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.49348230912477
      },
      "en": {
       "accuracy": 0.6794092093831451,
       "cc": 53.502802543117234,
       "cc_by_type": {
        "choice": 60.0200144006053,
        "noul": 62.58992805755397,
        "score": 37.89846517119242
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 53.502802543117234
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 63.40402392604676,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": -2.220446049250313e-14
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 63.40402392604676
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 44.990892531876135,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 44.990892531876135
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 78.90608875129,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 57.89473684210527
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 78.90608875129
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 29.073556797020483,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -25,
        "score": 62.5
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 29.073556797020483
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 71.59930223063503,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 42.85714285714285
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.59930223063503
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 49.82614067602153,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 80,
        "score": 26.66666666666665
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 49.82614067602153
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 4.761904761904745,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 14.285714285714235
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 4.761904761904745
      },
      "mixed": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 100
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 100
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 46.8975468975469,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 100
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.8975468975469
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 65.87093662962927,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 60.00000000000001,
        "score": 60.46511627906976
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 65.87093662962927
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 49.32372751408241,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": 60.00000000000001,
        "score": 51.61290322580645
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 49.32372751408241
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 52.97872340425531,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 50
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 52.97872340425531
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 53.260869565217384,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 100,
        "score": 24.999999999999968
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 53.260869565217384
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.58064516129032,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 50.00000000000002
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.58064516129032
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 27.49529190207156,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": 0
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 27.49529190207156
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.692,
       "cc": 60.21737882470005,
       "cc_by_type": {
        "choice": 60.21737882470005
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 60.21737882470005
      },
      "noul": {
       "accuracy": 0.8,
       "cc": 60.00000000000001,
       "cc_by_type": {
        "noul": 60.00000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 60.00000000000001
      },
      "score": {
       "accuracy": 0.5546666666666666,
       "cc": 39.98194945848379,
       "cc_by_type": {
        "score": 39.98194945848379
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 39.98194945848379
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 37.921568627450974,
       "cc_by_type": {
        "choice": 52,
        "noul": 50,
        "score": 11.76470588235292
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 37.921568627450974
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 25.81090407177364,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 33.33333333333333,
        "score": 2.220446049250313e-14
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 25.81090407177364
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 67.07317073170732,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 99.99999999999999
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 67.07317073170732
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 52.412280701754376,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": 100,
        "score": 62.499999999999986
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 52.412280701754376
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 49.426328502415465,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 100,
        "score": 6.250000000000022
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 49.426328502415465
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -43.18658280922435,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 33.33333333333333,
        "score": -200.00000000000009
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 35.40185304891187,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": 42.85714285714286,
        "score": 2.564102564102555
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 35.40185304891187
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 15.18709286471747,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -50,
        "score": 47.82608695652174
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 15.18709286471747
      },
      "insurance_claims": {
       "accuracy": 0.6666666666666666,
       "cc": 55.27777777777778,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 19.999999999999996,
        "score": 66.66666666666667
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 55.27777777777778
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 24.80911902753445,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 33.33333333333333,
        "score": 18.181818181818198
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 24.80911902753445
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 39.891159658601524,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 11.111111111111116,
        "score": 31.818181818181824
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 39.891159658601524
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": 5.60035842293907,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 12.50000000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 5.60035842293907
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 39.49456174024312,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -19.999999999999996,
        "score": 49.999999999999986
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 39.49456174024312
      },
      "model_routing": {
       "accuracy": 0.8433179723502304,
       "cc": 74.4609036796537,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 90.90909090909092,
        "score": 52.34375
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.4609036796537
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": -7.760340220541191,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": -11.764705882352944
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 80.32640525676737,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100,
        "score": 57.6923076923077
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 80.32640525676737
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 48.94716507292571,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 50,
        "score": 41.17647058823529
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 48.94716507292571
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 18.45410628019324,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": -19.999999999999996
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 18.45410628019324
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 97.36842105263158,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 92.10526315789474
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.36842105263158
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.50831193994927,
      84.98074981824452
     ],
     "I": [
      50.664175942542194,
      60.099057733365704
     ],
     "capability": [
      64.61139824588848,
      72.13730711245701
     ],
     "composite": null,
     "seed": 57
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.5161952575238,
     "capability": 69.29928706772574,
     "intelligence": 56.08237887792767
    },
    "per_type": {
     "choice": {
      "accuracy": 0.692,
      "calibration": 84.89511244131369,
      "cc": 60.21737882470005,
      "cc_by_type": {
       "choice": 60.21737882470005
      },
      "cc_tiered": 64.27000050619776,
      "cc_tiered_open": 72.53391464649734,
      "cc_tiered_sealed": 56.006086365898184,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.21737882470005
     },
     "noul": {
      "accuracy": 0.8,
      "calibration": 81.62460376654776,
      "cc": 60.00000000000001,
      "cc_by_type": {
       "noul": 60.00000000000001
      },
      "cc_tiered": 61.29187431497844,
      "cc_tiered_open": 68.99171842650105,
      "cc_tiered_sealed": 53.592030203455835,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 60.00000000000001
     },
     "score": {
      "accuracy": 0.5546666666666666,
      "calibration": 81.02886956471,
      "cc": 39.98194945848379,
      "cc_by_type": {
       "score": 39.98194945848379
      },
      "cc_tiered": 42.685261812606804,
      "cc_tiered_open": 49.50978713690577,
      "cc_tiered_sealed": 35.860736488307836,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 39.98194945848379
     }
    },
    "raw_on_A": {
     "calibration": 83.47832626238565,
     "capability": 71.17484534717096,
     "intelligence": 58.871364431956266
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 72.80793136991703,
    "Emphasis on Accuracy 60:20:20": 65.04815796310577,
    "Emphasis on Cost 20:20:60": 75.19830708539693,
    "Emphasis on Speed 20:60:20": 79.78996149386236,
    "Intelligence only": 56.08237887792767
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Phocinae / NovelNexus AI (novelnexusai@outlook.com)",
   "axes": {
    "calibration": 56.87058116402116,
    "cost": 95.30667675976518,
    "intelligence": 3.609767391262904,
    "speed": 76.50701878654316
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06613968253968247,
      "ece_score": 86.7720634920635,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 86.7720634920635,
      "score_before_time_limit": 86.7720634920635,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.9706666666666667,
      "brier": 0.24743832458666667,
      "ece": 0.08080160000000003,
      "n": 375,
      "score": 83.83967999999999,
      "score_before_time_limit": 83.83967999999999,
      "time_limit_failed_share": 0
     }
    },
    "score": 56.87058116402116
   },
   "capability": 30.240174277642033,
   "composite_ci95": {
    "A": [
     0.007598161089799276,
     0.224928678737021
    ],
    "B": [
     0.004911677300807149,
     0.1551384520354836
    ],
    "C": [
     0.0052765007568050535,
     0.15620047134515344
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference; self-hosted open weights, no public bookable tariff. A 144.3M ModernBERT encoder with no decoder lineage, so the base-size-encoder line is used unchanged - USD 0.005 per 1M input and 0.0 per 1M output, the line the PUBLISHED v1.6 field already carries on modernbert-jev-v2, tacet-sonata and watt-flash-0.1 (verified in their own registry entries; modernbert-jev-v2 was measured in v1.6.8 on the earlier v1.6-regular-20261010-a2 draw, while tacet-sonata and watt-flash-0.1 are v1.6.1 rows - none of the three was measured on THIS draw, which carries no ModernBERT row), backed by the dated 25 Sep 2026 DeepInfra snapshot sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f in which thenlper/gte-base, BAAI/bge-base-en-v1.5, intfloat/e5-base-v2 and sentence-transformers/all-mpnet-base-v2 all price at USD 0.005 per 1M input and 0 output - and The frozen pricing reference is unchanged. Output tokens are a literal 0 reported by the author's own server (an encoder read-out; nothing is generated). x this row's own measured input tokens, which are the author's server's own count. Estimated, not charged. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.001434 per 1,000 decisions (0.022x the official cost cap USD 0.064595), against USD 0.001440 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0014397966666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0014336531986531987
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Phocinae-Largha-150M-v1",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Sandy, the evaluator's own shared CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2), NOT a dedicated machine: other jobs ran beside it and the 1-minute load average was logged every 60 s into logs/CONDITIONS-<key>.jsonl - for this row's window 3.75 to 9.89, mean 5.81 over 12 samples, the quietest of the three new rows. The author's own phocinae-server 0.1.5 from PyPI, UNMODIFIED, run with their own documented line `PHOC_MODEL_DIR=<pinned snapshot> python -m phocinae.main` at PHOC_DEVICE=cpu, PHOC_THREADS=1 (their own documented CPU setting: 1.64 s per 5-decision case single-threaded), PHOC_EXTENSIONS=1 for their native option_scores, PHOC_PERM_AVG at its default 0 and no bearer token, inside a loopback-only network namespace with an outbound probe that failed as required and no credentials in the environment. Their package omits numpy from its declared dependencies although their own engine.py calls .numpy(); numpy 2.5.4 was installed so their code could run, and no file of theirs was edited. OUR shim (r107/src/phoc_shim_r107.py, byte-identical to the already source-reviewed r92/src/phoc_shim_r92.py, sha256 631cb60dde86b8e609382a32900db5e4ddc9fa9b2fef39bdb7b022dff558775e) is the only translation layer in front of it, because their published API carries no question text - see the mapping disclosure. Speed is a loaded-shared-CPU figure; the standard self-hosted x2 + 0.15 s adjustment is applied on top and the raw p50/p95 are kept. Re-measured for this release on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b) under the pre-registration, at USD 0 on Sandy’s own CPU, 1 thread, 2026-10-13 05:59:36Z–06:11:07Z, 1-minute load average 3.75–9.89 (mean 5.81, 12 samples). DISCLOSED MAPPING: choice options are sent as \"<label>: <description>\" in the item’s own criteria order and the author’s option_scores are mapped back onto the bare labels by position; for noul only the author’s p(true) is forwarded and the stock adapter derives p(no) = 1 - p(yes), so their own rounded p(false) is unused. The author’s API answers score on a fixed 9-level scale 2..10 against items with 4 or 5 levels and forbids score options, so no identity mapping exists and this row uses pre-committed unsupported-native-score handling fixed before any measurement; all 375 score items of this draw are recorded as item-level misses (HTTP 422), and support.score is left at \"native\" rather than \"unsupported\" so that the type is still weighted and the misses count — the stricter of the two treatments, and the same rule every other row gets.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 4.178627926416878,
    "I_sealed": 3.04090685610893,
    "base": 3.609767391262904,
    "excess": -8.701805700047764,
    "gap": 1.137721070307948,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 12.535883779250636,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 2.7027027027027115,
       "hard": 5.630701478452351,
       "judge": 16.94133157547792,
       "standard": 24.65466722478024
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100,
       "hard": -100,
       "judge": -100,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -163.63636363636363,
       "hard": -161.1111111111111,
       "judge": -213.55932203389827,
       "standard": -355.5555555555556
      }
     },
     "sealed|choice": {
      "cc": 9.122720568326791,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": -4.828462515883092,
       "hard": 9.448363594310154,
       "judge": 10.969637610186114,
       "standard": 12.676650495676029
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -100,
       "hard": -96.36363636363636,
       "judge": -93.84615384615384,
       "standard": -97.01492537313433
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": -184.21052631578956,
       "hard": -207.887323943662,
       "judge": -209.375,
       "standard": -204.54545454545453
      }
     }
    }
   },
   "jevbench_score": 0.06552619942227712,
   "key": "phocinae-largha-150m-v1",
   "last_measured_on": "2026-10-13",
   "licence": "Apache-2.0 (LICENSE + NOTICE in the weights repo; the author's phocinae-server is Apache-2.0 too)",
   "listing": "ranked",
   "measured_in": "v1.6.15",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Phocinae/Phocinae-Largha-150M-v1@7070e070fec3407061e76de42d7d87240ff5b74b",
   "noul_decisive": {
    "acc_among_decisive": 0.5454545454545454,
    "decisive": 11,
    "decisive_rate": 0.029333333333333333,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 56.87058116402116,
     "cost": 95.30667675976518,
     "intelligence": 3.609767391262904,
     "speed": 77.11764018052709
    },
    "board_release": "v1.6.43",
    "capability": 30.240174277642033,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 130,
    "capability_rank": 159,
    "composite_ci95": {
     "A": [
      0.007599423981329298,
      0.22503410752798958
     ],
     "B": [
      0.004912099447884301,
      0.15517856706924435
     ],
     "C": [
      0.005277377764812013,
      0.15627368578332607
     ]
    },
    "jevbench_score": 0.06554752053348605,
    "measured_in": "v1.6.15",
    "method": "v1.6",
    "open_board_rank": 181,
    "rank": 225,
    "ranks": {
     "A": 225,
     "B": 225,
     "C": 225,
     "capability": 228
    },
    "scores": {
     "A": 0.06554752053348605,
     "B": 0.04379268583036826,
     "C": 0.045519111481587535
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.0369279810023726,
     "p50_s_raw": 0.44346399050118634,
     "p95_s_adjusted": 1.8727826966009158,
     "p95_s_raw": 0.8613913483004579
    },
    "views": {
     "Balanced 33:33:33": 0.052037415444369725,
     "Emphasis on Accuracy 60:20:20": 0.030496990613082496,
     "Emphasis on Cost 20:20:60": 0.08106754873380384,
     "Emphasis on Speed 20:60:20": 0.07983832019244448,
     "Intelligence only": 0.018814714977998477
    }
   },
   "rank": 267,
   "ranked": true,
   "ranks": {
    "A": 267,
    "B": 267,
    "C": 267,
    "capability": 269
   },
   "repo": "https://huggingface.co/Phocinae/Phocinae-Largha-150M-v1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.06552619942227712,
    "B": 0.04378507107044704,
    "C": 0.04550430515435912
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.1184825160002219,
    "p50_s_raw": 0.484241258000111,
    "p95_s_adjusted": 1.9983378840101065,
    "p95_s_raw": 0.9241689420050533
   },
   "status": {
    "answered_ok": 1125,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.13636363636363635,
       "cc": -169.44444444444443,
       "cc_by_type": {
        "choice": -8.333333333333334,
        "noul": -100,
        "score": -400
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.22772277227722773,
       "cc": -92.00468363379287,
       "cc_by_type": {
        "choice": 42.73594909862143,
        "noul": -100,
        "score": -218.75
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.12871287128712872,
       "cc": -112.19743821369839,
       "cc_by_type": {
        "choice": 2.4390243902439046,
        "noul": -85.18518518518519,
        "score": -253.84615384615387
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.125,
       "cc": -78.57142857142856,
       "cc_by_type": {
        "choice": 14.285714285714292,
        "noul": -100,
        "score": -149.99999999999994
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.18888888888888888,
       "cc": -91.6056349618032,
       "cc_by_type": {
        "choice": 11.934552454282981,
        "noul": -94.5945945945946,
        "score": -192.156862745098
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.11182108626198083,
       "cc": -110.9060491326935,
       "cc_by_type": {
        "choice": 7.28185260191953,
        "noul": -100,
        "score": -240
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -134.40860215053763,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.18518518518518517,
       "cc": -100.9327442317133,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100,
        "score": -228.57142857142856
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.13454545454545455,
       "cc": -96.41572945990254,
       "cc_by_type": {
        "choice": 5.11900880339095,
        "noul": -100,
        "score": -194.3661971830986
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.1511627906976744,
       "cc": -133.33333333333334,
       "cc_by_type": {
        "choice": 0,
        "noul": -80,
        "score": -320
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.25,
       "cc": 4.127829560585888,
       "cc_by_type": {
        "choice": 4.127829560585888
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 4.127829560585888
      },
      "perturb_pair": {
       "accuracy": 0.203125,
       "cc": -107.95410795410794,
       "cc_by_type": {
        "choice": 6.906906906906911,
        "noul": -100,
        "score": -230.76923076923072
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.14545454545454545,
       "cc": -83.8117647058823,
       "cc_by_type": {
        "choice": 11.764705882352976,
        "noul": -100,
        "score": -163.19999999999985
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -95.97069597069596,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": -100,
        "score": -230.76923076923075
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0,
       "cc": -63.15789473684211,
       "cc_by_type": {
        "choice": -26.315789473684216,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.15384615384615385,
       "cc": -107.64976958525347,
       "cc_by_type": {
        "choice": 17.050691244239633,
        "noul": -100,
        "score": -240
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.125,
       "cc": -101.42362447095036,
       "cc_by_type": {
        "choice": 4.424778761061946,
        "noul": -100,
        "score": -208.69565217391303
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "el": {
       "accuracy": 0,
       "cc": -78.02607076350093,
       "cc_by_type": {
        "choice": -34.07821229050279,
        "noul": -100,
        "score": -100
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.158992180712424,
       "cc": -95.95231542756851,
       "cc_by_type": {
        "choice": 9.715527025542716,
        "noul": -95.68345323741008,
        "score": -201.88902007083814
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.1935483870967742,
       "cc": -140.48555800786872,
       "cc_by_type": {
        "choice": 31.484502446982063,
        "noul": -100,
        "score": -352.94117647058823
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -66.12021857923497,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": -100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.07142857142857142,
       "cc": -98.45201238390091,
       "cc_by_type": {
        "choice": -5.882352941176468,
        "noul": -100,
        "score": -189.47368421052627
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.14285714285714285,
       "cc": -152.93296089385476,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100,
        "score": -375
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.2,
       "cc": -113.77352187952344,
       "cc_by_type": {
        "choice": 15.822291504286824,
        "noul": -100,
        "score": -257.1428571428571
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.04,
       "cc": -118.12549642573471,
       "cc_by_type": {
        "choice": -14.376489277204131,
        "noul": -100,
        "score": -240
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0,
       "cc": -115.87301587301586,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": -214.28571428571428
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.21428571428571427,
       "cc": -61.11111111111111,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -100,
        "score": -100
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.1,
       "cc": -114.63203463203463,
       "cc_by_type": {
        "choice": -3.8961038961038925,
        "noul": -100,
        "score": -240
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.1794871794871795,
       "cc": -81.20540503301875,
       "cc_by_type": {
        "choice": 23.82564536606009,
        "noul": -100,
        "score": -167.44186046511635
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.10714285714285714,
       "cc": -102.78860227726348,
       "cc_by_type": {
        "choice": 4.537418974661164,
        "noul": -100,
        "score": -212.9032258064516
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -67.37588652482269,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100,
        "score": -100
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.15384615384615385,
       "cc": -103.98550724637681,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": -225
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0,
       "cc": -93.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": -100,
        "score": -150
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.2,
       "cc": -125.42372881355932,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.296,
       "cc": 9.068294456457268,
       "cc_by_type": {
        "choice": 9.068294456457268
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 9.068294456457268
      },
      "noul": {
       "accuracy": 0.016,
       "cc": -96.8,
       "cc_by_type": {
        "noul": -96.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0,
       "cc": -203.0685920577614,
       "cc_by_type": {
        "score": -203.0685920577614
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.045454545454545456,
       "cc": -107.92156862745098,
       "cc_by_type": {
        "choice": -12,
        "noul": -100,
        "score": -211.76470588235296
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.125,
       "cc": -86.38318203535594,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -100,
        "score": -184.61538461538458
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.42857142857142855,
       "cc": -110.84010840108401,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.11764705882352941,
       "cc": -110.08771929824563,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": -225
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.17647058823529413,
       "cc": -91.48550724637681,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": -187.5
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0,
       "cc": -341.9287211740041,
       "cc_by_type": {
        "choice": -25.786163522012576,
        "noul": -100,
        "score": -899.9999999999998
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.26666666666666666,
       "cc": -67.62192056309704,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -100,
        "score": -153.84615384615387
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.13636363636363635,
       "cc": -83.53279806089985,
       "cc_by_type": {
        "choice": 5.9233449477351945,
        "noul": -100,
        "score": -156.52173913043475
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.13333333333333333,
       "cc": -72.22222222222223,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -100,
        "score": -133.33333333333334
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.045454545454545456,
       "cc": -93.08935176172865,
       "cc_by_type": {
        "choice": -15.631691648822272,
        "noul": -100,
        "score": -163.63636363636368
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.12,
       "cc": -76.46229739252993,
       "cc_by_type": {
        "choice": 6.976744186046504,
        "noul": -100,
        "score": -136.36363636363632
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.1,
       "cc": -107.25806451612902,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100,
        "score": -225
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.1111111111111111,
       "cc": -91.82128385583282,
       "cc_by_type": {
        "choice": 7.869481765834927,
        "noul": -100,
        "score": -183.3333333333334
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.1152073732718894,
       "cc": -107.73538961038962,
       "cc_by_type": {
        "choice": 11.168831168831192,
        "noul": -100,
        "score": -234.37500000000006
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0
      },
      "moderation_trust_safety": {
       "accuracy": 0.16666666666666666,
       "cc": -89.5337021564864,
       "cc_by_type": {
        "choice": 7.869481765834927,
        "noul": -100,
        "score": -176.47058823529412
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.1111111111111111,
       "cc": -87.27233769016495,
       "cc_by_type": {
        "choice": -0.2785515320334252,
        "noul": -100,
        "score": -161.53846153846146
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.05555555555555555,
       "cc": -98.23239640683859,
       "cc_by_type": {
        "choice": -18.226600985221676,
        "noul": -100,
        "score": -176.47058823529412
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.043478260869565216,
       "cc": -91.98067632850244,
       "cc_by_type": {
        "choice": -15.942028985507239,
        "noul": -100,
        "score": -160.00000000000006
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.125,
       "cc": -74.13373290878444,
       "cc_by_type": {
        "choice": 3.914590747330961,
        "noul": -100,
        "score": -126.31578947368425
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      51.501811957671954,
      59.20755983068783
     ],
     "I": [
      1.71972943365577,
      5.572698362718295
     ],
     "capability": [
      27.217574553966454,
      31.914331612070672
     ],
     "composite": {
      "A": [
       0.007598161089799276,
       0.224928678737021
      ],
      "B": [
       0.004911677300807149,
       0.1551384520354836
      ],
      "C": [
       0.0052765007568050535,
       0.15620047134515344
      ]
     },
     "seed": 57
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 56.87058116402116,
     "capability": 30.240174277642033,
     "intelligence": 3.609767391262904
    },
    "per_type": {
     "choice": {
      "accuracy": 0.296,
      "calibration": 86.7720634920635,
      "cc": 9.068294456457268,
      "cc_by_type": {
       "choice": 9.068294456457268
      },
      "cc_tiered": 10.829302173788713,
      "cc_tiered_open": 12.535883779250636,
      "cc_tiered_sealed": 9.122720568326791,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 9.068294456457268
     },
     "noul": {
      "accuracy": 0.016,
      "calibration": 83.83967999999999,
      "cc": -96.8,
      "cc_by_type": {
       "noul": -96.8
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0,
      "calibration": null,
      "cc": -203.0685920577614,
      "cc_by_type": {
       "score": -203.0685920577614
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 53.88631212892281,
     "capability": 28.527882250386632,
     "intelligence": 3.169452371850454
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 1,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.05201949869554396,
    "Emphasis on Accuracy 60:20:20": 0.03049329752256628,
    "Emphasis on Cost 20:20:60": 0.08104145820610814,
    "Emphasis on Speed 20:60:20": 0.07976245200357382,
    "Intelligence only": 0.018814714977998477
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Kailune AI / @chi2liu (GitHub issue fstandhartinger/jevbench#145)",
   "axes": {
    "calibration": 84.15501405892886,
    "cost": 63.52461077797332,
    "intelligence": 28.75448619260912,
    "speed": 92.67163234372039
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07807774485134686,
      "ece_score": 84.38445102973063,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 84.38445102973063,
      "score_before_time_limit": 84.38445102973063,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.41018766756032177,
      "brier": 0.16186851482302253,
      "ece": 0.08062076189083324,
      "n": 373,
      "score": 83.87584762183336,
      "score_before_time_limit": 83.87584762183336,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9408602150537635,
      "interval90_width": 0.5562275985663083,
      "n": 372,
      "nrps": 0.12208071152631676,
      "score": 84.20474352522257,
      "score_before_time_limit": 84.20474352522257,
      "time_limit_failed_share": 0,
      "top_ece": 0.09691220898461597
     }
    },
    "score": 84.15501405892886
   },
   "capability": 56.45475012576899,
   "composite_ci95": {
    "A": [
     11.391972969610558,
     26.728013798311753
    ],
    "B": [
     9.375501093997054,
     23.291940599860233
    ],
    "C": [
     7.911092340007333,
     18.561120693272045
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), size-class reference, carried over unchanged from PREREG-R36.md section 3.2. The UNMODIFIED frozen pricing_v15 path is asserted: openbmb/MiniCPM5-2B is in UNLISTED_BASE_MODELS - explicitly reviewed against both 25 Sep 2026 catalogs and not sold - so reference() returns no market reference rather than raising, and no exact base-model floor applies. The board already prices a row on THE SAME BASE this way: decision-2b (FlyMy.AI, openbmb/MiniCPM5-2B + LoRA and pointer head) carries the DeepInfra 2-4B size-class reference list price of USD 0.03 per 1M input and USD 0.0 per 1M output from the frozen 25 Sep 2026 snapshot (sha256 84eb7926acd4e93b71d695be24898e2ca999b20011d1b66cc0348ce7c6c3546f). pjev-2b is the same shape: 2.52B, one forward pass with logits_to_keep=1, NOTHING GENERATED - the author's serve.py reports usage.output_tokens as a literal 0, not a decision count - so the output term is 0.0, not 0.15. Multiplied by this row's own measured input tokens. The frozen pricing reference is unchanged. Free weights are never cost 100 and never cost 0. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.016438 per 1,000 decisions (0.254x the official cost cap USD 0.064595), against USD 0.016274 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.01627352
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01643789898989899
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "pjev-2b (Kailune AI, frozen MiniCPM5-2B + merged LoRA r64, letter-logit readout, T=1 stub)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Verda RTX 6000 Ada (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-02, one dedicated GPU instance, network none (proven), evaluator-controlled self-hosted deployment. The author's own serve.py / decision_core.py / letter_adapter.py, unmodified, at their declared GPU configuration: `python3 serve.py --model <pinned snapshot> --device cuda --host 127.0.0.1 --port <p>` at their own defaults (--max-tokens 16000), bfloat16 as hard-coded by the author, WITHOUT our earlier float32 patch of letter_adapter.py (PJEV_DTYPE; the flag is forbidden by the gate and absent from the launched argv/env). No deviation from the author's declared configuration other than the hardware. letter_adapter.dist() raises rather than truncating a prompt over the author's 16,000-token contract and serve.py maps that to HTTP 422, so the draw's 15 longest items are honest refusals scored as wrong (the same 15 items as in the CPU row). Speed is a dedicated-GPU figure, measured one request at a time; the standard self-hosted x2 + 0.15 s adjustment applies and raw p50/p95 are kept. RE-MEASURE ON THE SAME ITEMS: this GPU row REPLACES the earlier loaded-CPU row in full (scores AND times); the GPU times are not combined with the CPU scores. The same 1,500 items of draw v1.6-regular-20261012 (input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b), the same pinned weights. Pod: Verda 1RTX6000ADA.10V (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-02, torn down after the run (22 Oct 2026). The scoring run had no network: container NetworkMode none, only the loopback interface, cut proven by three failed outbound probes before the first sealed byte; read-only root, all capabilities dropped. The earlier CPU measurement of this row was its first use of the weights on this draw; this is the second and last.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 32.60865298275894,
    "I_sealed": 24.9003194024593,
    "base": 28.75448619260912,
    "excess": -2.13119319005607,
    "gap": 7.708333580299641,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 59.27670193309456,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 50.92796476879522,
       "judge": 52.537903757415954,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 11.725327812284332,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 13.043478260869556,
       "judge": 0,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 26.823929202897954,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.07732888401458,
       "hard": 16.856096268424558,
       "judge": 33.35520857550455,
       "standard": 20.335976172376547
      }
     },
     "sealed|choice": {
      "cc": 52.21091228947275,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 67.5984752223634,
       "hard": 50.04047646582629,
       "judge": 49.75514201762979,
       "standard": 52.5416578780848
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 10.344827586206895,
       "hard": -3.6363636363636376,
       "judge": -10.769230769230765,
       "standard": 7.462686567164178
      }
     },
     "sealed|score": {
      "cc": 22.49004591790515,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 20.380230279018463,
       "hard": 25.193643152818733,
       "judge": 19.747097400249913,
       "standard": 22.252182044004165
      }
     }
    }
   },
   "jevbench_score": 18.074372740962183,
   "key": "pjev-2b",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 on the artifact (not gated, not private, not disabled). The three serving files are redistributed unmodified from the Eikos project (github.com/caiovicentino/eikos) under MIT, Copyright (c) 2026 Caio Vicentino; NOTICE and LICENSE-eikos are both present at the pinned revision.",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Kailune-AI/pjev-2b@ad60e59c7aa1f6a545c563bc404853f9a8067908",
   "noul_decisive": {
    "acc_among_decisive": 0.8772727272727273,
    "decisive": 220,
    "decisive_rate": 0.5898123324396782,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.466579728622,
     "cost": 63.52461077797332,
     "intelligence": 28.96851828460498,
     "speed": 49.10963929819891
    },
    "board_release": "v1.6.43",
    "capability": 57.71754900661349,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      10.021228315163317,
      23.41864571116037
     ],
     "B": [
      8.582473580841295,
      21.268305262838254
     ],
     "C": [
      6.959186329974525,
      16.26294841052804
     ]
    },
    "jevbench_score": 15.759681433648748,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": 118,
    "rank": 156,
    "ranks": {
     "A": 156,
     "B": 155,
     "C": 156,
     "capability": 145
    },
    "scores": {
     "A": 15.759681433648748,
     "B": 13.872903874213488,
     "C": 10.944223217811633
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 20.27664159500855,
     "p50_s_raw": 10.063320797504275,
     "p95_s_adjusted": 60.539669460893485,
     "p95_s_raw": 30.194834730446743
    },
    "views": {
     "Balanced 33:33:33": 13.755306662189126,
     "Emphasis on Accuracy 60:20:20": 11.592783816847163,
     "Emphasis on Cost 20:20:60": 15.856729233868942,
     "Emphasis on Speed 20:60:20": 14.54072118964529,
     "Intelligence only": 9.380636713101017
    }
   },
   "rank": 192,
   "ranked": true,
   "ranks": {
    "A": 192,
    "B": 193,
    "C": 193,
    "capability": 190
   },
   "remeasure": {
    "disclosure": "Re-measure on the same items: this GPU row replaces the earlier loaded-CPU row of the same weights in full (scores and times) and is the second and last use of these weights on draw v1.6-regular-20261012.",
    "draw": "v1.6-regular-20261012",
    "kind": "re-measure on the same items",
    "replaces": {
     "board_release": "v1.6.43",
     "endpoint_kind": "cpu",
     "in_full": "scores and times"
    }
   },
   "remeasured_in": "v1.7.0",
   "repo": "https://huggingface.co/Kailune-AI/pjev-2b",
   "scores": {
    "A": 18.074372740962183,
    "B": 15.315756542769432,
    "C": 12.551647736779294
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22076407499994274,
    "p50_s_raw": 0.035382037499971375,
    "p95_s_adjusted": 0.2448546542999793,
    "p95_s_raw": 0.04742732714998965
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6363636363636364,
       "cc": 47.192625784211685,
       "cc_by_type": {
        "choice": 41.666666666666664,
        "noul": 100,
        "score": -0.08878931403160095
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 47.192625784211685
      },
      "calibration_gen": {
       "accuracy": 0.45544554455445546,
       "cc": 2.779417852609811,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": -60,
        "score": 12.876959814457212
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 2.779417852609811
      },
      "composite_gen": {
       "accuracy": 0.5148514851485149,
       "cc": 14.586734299630178,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": -25.92592592592593,
        "score": 18.46661662969451
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 14.586734299630178
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 69.23140098064862,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 47.69420294194586
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 69.23140098064862
      },
      "jaggedness": {
       "accuracy": 0.35,
       "cc": -13.833821090282077,
       "cc_by_type": {
        "choice": 29.258902791145346,
        "noul": -72.97297297297297,
        "score": 2.212606910981396
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.6996805111821086,
       "cc": 48.30203158352211,
       "cc_by_type": {
        "choice": 61.96178568283877,
        "noul": 52.57731958762886,
        "score": 30.366989480098695
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 48.30203158352211
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -9.026636236663187,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -50,
        "score": 9.518029434340336
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6072727272727273,
       "cc": 31.8492008743559,
       "cc_by_type": {
        "choice": 61.85197261167266,
        "noul": -2.777777777777779,
        "score": 36.47340778917284
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 31.8492008743559
      },
      "negation_pair_gen": {
       "accuracy": 0.5581395348837209,
       "cc": 14.453640352520686,
       "cc_by_type": {
        "choice": 56.66666666666668,
        "noul": -19.999999999999996,
        "score": 6.694254390895371
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 14.453640352520686
      },
      "option_order": {
       "accuracy": 0.5416666666666666,
       "cc": 41.41145139813582,
       "cc_by_type": {
        "choice": 41.41145139813582
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 41.41145139813582
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 64.47617964735888,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 80,
        "score": 46.46157197510967
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 64.47617964735888
      },
      "usecase": {
       "accuracy": 0.5681818181818182,
       "cc": 31.502274259633936,
       "cc_by_type": {
        "choice": 60.78431372549021,
        "noul": 4.166666666666674,
        "score": 29.555842386744924
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 31.502274259633936
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 23.320769294368613,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50,
        "score": -3.8472159264179684
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 23.320769294368613
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": -10.526315789473685,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 37.12677894367112,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 44.71367016434669
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 37.12677894367112
      },
      "de": {
       "accuracy": 0.71875,
       "cc": 52.26829138448147,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 25,
        "score": 63.663281233090416
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 52.26829138448147
      },
      "el": {
       "accuracy": 0.4,
       "cc": 23.361075096743434,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -33.333333333333336,
        "score": 70.45566476881504
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 23.361075096743434
      },
      "en": {
       "accuracy": 0.5560382276281495,
       "cc": 24.818610987257575,
       "cc_by_type": {
        "choice": 51.45287462930642,
        "noul": 1.4388489208633004,
        "score": 21.564109411603006
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 24.818610987257575
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 52.511825617037566,
       "cc_by_type": {
        "choice": 100,
        "noul": 75,
        "score": -17.464523148887313
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 52.511825617037566
      },
      "fi": {
       "accuracy": 0.25,
       "cc": 11.324912795495031,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 65.66872745752335
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 11.324912795495031
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 47.62609454295699,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 64.05475421710626
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 47.62609454295699
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": 8.604361095662563,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -50,
        "score": 9.332636359613389
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 8.604361095662563
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 57.17303362006391,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": -0.4216629745705136
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 57.17303362006391
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 26.880921629481804,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 17.83100952704748
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 26.880921629481804
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 44.80451619025891,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100,
        "score": -21.142006984778817
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 44.80451619025891
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 72.90992448106408,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 85.3964401098589
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 72.90992448106408
      },
      "nl": {
       "accuracy": 0.4,
       "cc": 15.56120940445134,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -33.333333333333336,
        "score": 57.939039468765266
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 15.56120940445134
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 35.20767856477335,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 0,
        "score": 28.475342084502053
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 35.20767856477335
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 48.195957525907836,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 16.40873291950311
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 48.195957525907836
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": 1.7997955219120019,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": -19.999999999999996,
        "score": 53.058961033821106
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 1.7997955219120019
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 31.137380604564715,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0,
        "score": 15.151272248476754
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 31.137380604564715
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.71888881665404,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 59.414730966091156
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.71888881665404
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 4.828693723155223,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -26.756856683641672
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 4.828693723155223
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.644,
       "cc": 54.01748981036759,
       "cc_by_type": {
        "choice": 54.01748981036759
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 54.01748981036759
      },
      "noul": {
       "accuracy": 0.5146666666666667,
       "cc": 2.9333333333333433,
       "cc_by_type": {
        "noul": 2.9333333333333433
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 2.9333333333333433
      },
      "score": {
       "accuracy": 0.4746666666666667,
       "cc": 23.713042630287838,
       "cc_by_type": {
        "score": 23.713042630287838
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 23.713042630287838
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": 6.415810838049524,
       "cc_by_type": {
        "choice": 52,
        "noul": -50,
        "score": 17.247432514148574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 6.415810838049524
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 20.933052765336722,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 33.33333333333333,
        "score": -14.633553919310739
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 20.933052765336722
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 80.65848421901062,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 74.49577786028385
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 80.65848421901062
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -2.143072671679962,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -33.333333333333336,
        "score": 8.775460347533215
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 24.66934193937999,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": 33.33333333333333,
        "score": 13.138460600748669
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 24.66934193937999
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -35.65918836172522,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -100,
        "score": -56.66309967637062
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 34.45878035079783,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -14.28571428571429,
        "score": 37.269898475362695
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 34.45878035079783
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": 0.7336197621111668,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50,
        "score": 25.37159099365057
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0.7336197621111668
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 25.002590599297374,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -60,
        "score": 55.84110513122545
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 25.002590599297374
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": -0.6302270781537042,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": -33.333333333333336,
        "score": 8.530446531420399
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 23.73256258574339,
       "cc_by_type": {
        "choice": 65.11627906976744,
        "noul": 11.111111111111116,
        "score": -5.029702423648397
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 23.73256258574339
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": -5.525839680265041,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100,
        "score": 15.680545475333918
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 33.3680599381754,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -19.999999999999996,
        "score": 43.13680937306745
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 33.3680599381754
      },
      "model_routing": {
       "accuracy": 0.7419354838709677,
       "cc": 49.458148272289016,
       "cc_by_type": {
        "choice": 77.7922077922078,
        "noul": 54.54545454545454,
        "score": 16.036782479204703
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 49.458148272289016
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 28.4251098729428,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": 0,
        "score": 54.37321829445221
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 28.4251098729428
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 45.34747123856698,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 36.13526422637875
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 45.34747123856698
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 20.504013092940667,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50,
        "score": 41.06868952512741
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 20.504013092940667
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 22.208565117404365,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 0,
        "score": 24.596709844966714
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 22.208565117404365
      },
      "search_retrieval": {
       "accuracy": 0.7083333333333334,
       "cc": 55.98773712725733,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": 50,
        "score": 49.99168113266167
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 55.98773712725733
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.72190811066586,
      85.45748339862506
     ],
     "I": [
      23.94121773861638,
      33.805531403151264
     ],
     "capability": [
      51.96793745679222,
      58.52381546627118
     ],
     "composite": {
      "A": [
       11.391972969610558,
       26.728013798311753
      ],
      "B": [
       9.375501093997054,
       23.291940599860233
      ],
      "C": [
       7.911092340007333,
       18.561120693272045
      ]
     },
     "seed": 58
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.15501405892886,
     "capability": 56.45475012576899,
     "intelligence": 28.75448619260912
    },
    "per_type": {
     "choice": {
      "accuracy": 0.644,
      "calibration": 84.38445102973063,
      "cc": 54.01748981036759,
      "cc_by_type": {
       "choice": 54.01748981036759
      },
      "cc_tiered": 55.74380711128366,
      "cc_tiered_open": 59.27670193309456,
      "cc_tiered_sealed": 52.21091228947275,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.01748981036759
     },
     "noul": {
      "accuracy": 0.5146666666666667,
      "calibration": 83.87584762183336,
      "cc": 2.9333333333333433,
      "cc_by_type": {
       "noul": 2.9333333333333433
      },
      "cc_tiered": 5.862663906142166,
      "cc_tiered_open": 11.725327812284332,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 2.9333333333333433
     },
     "score": {
      "accuracy": 0.4746666666666667,
      "calibration": 84.20474352522257,
      "cc": 23.713042630287838,
      "cc_by_type": {
       "score": 23.713042630287838
      },
      "cc_tiered": 24.656987560401554,
      "cc_tiered_open": 26.823929202897954,
      "cc_tiered_sealed": 22.49004591790515,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 23.713042630287838
     }
    },
    "raw_on_A": {
     "calibration": 79.07470646766168,
     "capability": 54.07122546865802,
     "intelligence": 29.067744469654354
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.18310508373799,
    "Emphasis on Accuracy 60:20:20": 12.636306415598074,
    "Emphasis on Cost 20:20:60": 17.820604749959447,
    "Emphasis on Speed 20:60:20": 19.94946868565402,
    "Intelligence only": 9.509919186674669
   }
  },
  {
   "adapter_id": "images/plumb-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "unknown",
   "axes": {
    "calibration": 87.80311208802716,
    "cost": 77.61552273233744,
    "intelligence": 42.83684414340664,
    "speed": 93.68902806065091
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.061775972135365,
      "ece_score": 87.644805572927,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 87.41108609139918,
      "score_before_time_limit": 87.644805572927,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "noul": {
      "abstention_rate": 0.29333333333333333,
      "brier": 0.12494153723477718,
      "ece": 0.06617846187638739,
      "n": 375,
      "score": 86.76430762472252,
      "score_before_time_limit": 86.76430762472252,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9573333333333334,
      "interval90_width": 0.49066666666666664,
      "n": 375,
      "nrps": 0.09744572815253887,
      "score": 89.2339425479598,
      "score_before_time_limit": 89.2339425479598,
      "time_limit_failed_share": 0,
      "top_ece": 0.058937710444132485
     }
    },
    "score": 87.80311208802716
   },
   "capability": 65.3199781157169,
   "class": "unclassified",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 921,181 input + 0 output tokens over 1,624 decisions = 567.229679803 tokens/decision. GPU-time cost USD 0.00557377254813054/1,000; old list-price cost USD 0.0170168903940887/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00557377254813054/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005573772548130538
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 2,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Plumb-4B (crh225, JevK5 v0.2 + LoRA)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 44.9020239650968,
    "I_sealed": 40.77166432171647,
    "base": 42.83684414340664,
    "excess": 1.5569720009420869,
    "gap": 4.130359643380331,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 62.95060504058115,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 52.815350739226176,
       "judge": 72.31377719182596,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 36.11249137336094,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 35.64297548134833,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.38519253276966,
       "hard": 30.154558838586563,
       "judge": 43.95614933948649,
       "standard": 21.77893945395395
      }
     },
     "sealed|choice": {
      "cc": 58.16335833730948,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 57.239819004524875,
       "judge": 57.16945996275605,
       "standard": 57.876585786426915
      }
     },
     "sealed|noul": {
      "cc": 23.383111546129097,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 56.00000000000001,
       "hard": 4.854368932038833,
       "judge": 36.8421052631579,
       "standard": 23.943661971830977
      }
     },
     "sealed|score": {
      "cc": 40.76852308171083,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 65.90236427761113,
       "hard": 40.87931496804801,
       "judge": 36.35703355260749,
       "standard": 34.597253004741354
      }
     }
    }
   },
   "jevbench_score": 50.367731512900676,
   "key": "plumb-4b",
   "last_measured_on": "2026-10-02",
   "licence": "not recorded",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9132075471698113,
    "decisive_rate": 0.7066666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.88101858186977,
     "cost": 63.07359395281681,
     "intelligence": 42.9626698747687,
     "speed": 93.83356539601934
    },
    "board_release": "v1.6.43",
    "capability": 65.42184422831923,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 37,
    "capability_rank": 58,
    "composite_ci95": {
     "A": [
      34.218232721078074,
      62.094303811225494
     ],
     "B": [
      30.257952435880583,
      57.342398440511
     ],
     "C": [
      23.762661611859777,
      43.121044313351035
     ]
    },
    "jevbench_score": 48.28224126395837,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 43,
    "rank": 67,
    "ranks": {
     "A": 67,
     "B": 69,
     "C": 73,
     "capability": 100
    },
    "scores": {
     "A": 48.28224126395837,
     "B": 43.7169924526014,
     "C": 33.529334211082194
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.18999176411889493,
     "p50_s_raw": 0.019995882059447467,
     "p95_s_adjusted": 0.21772519345395266,
     "p95_s_raw": 0.033862596726976335
    },
    "views": {
     "Balanced 33:33:33": 44.487894416524036,
     "Emphasis on Accuracy 60:20:20": 38.31837801553863,
     "Emphasis on Cost 20:20:60": 45.2973332743533,
     "Emphasis on Speed 20:60:20": 51.919511056412496,
     "Intelligence only": 31.720043803902097
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 122,
   "ranked": true,
   "ranks": {
    "A": 122,
    "B": 123,
    "C": 128,
    "capability": 129
   },
   "repo": "https://huggingface.co/crh225/plumb-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 50.367731512900676,
    "B": 44.955799956646935,
    "C": 34.977591328403236
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.1921826101373881,
    "p50_s_raw": 0.021091305068694055,
    "p95_s_adjusted": 0.22252720159012823,
    "p95_s_raw": 0.03626360079506411
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.6386296587994,
      88.25546310062707
     ],
     "I": [
      37.12848978026168,
      47.77693524356761
     ],
     "capability": [
      60.13168890043501,
      67.144764362578
     ],
     "composite": null,
     "seed": 85
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.80311208802716,
     "capability": 65.3199781157169,
     "intelligence": 42.83684414340664
    },
    "per_type": {
     "choice": {
      "accuracy": 0.684,
      "calibration": 87.41108609139918,
      "cc": 59.58811325938065,
      "cc_by_type": {
       "choice": 59.58811325938065
      },
      "cc_tiered": 60.556981688945314,
      "cc_tiered_open": 62.95060504058115,
      "cc_tiered_sealed": 58.16335833730948,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.58811325938065
     },
     "noul": {
      "accuracy": 0.6453333333333333,
      "calibration": 86.76430762472252,
      "cc": 29.066666666666663,
      "cc_by_type": {
       "noul": 29.066666666666663
      },
      "cc_tiered": 29.74780145974502,
      "cc_tiered_open": 36.11249137336094,
      "cc_tiered_sealed": 23.383111546129097,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 29.066666666666663
     },
     "score": {
      "accuracy": 0.576,
      "calibration": 89.2339425479598,
      "cc": 41.48168877172061,
      "cc_by_type": {
       "score": 41.48168877172061
      },
      "cc_tiered": 38.20574928152958,
      "cc_tiered_open": 35.64297548134833,
      "cc_tiered_sealed": 40.76852308171083,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 41.48168877172061
     }
    },
    "raw_on_A": {
     "calibration": 82.08788861322404,
     "capability": 66.01184998894868,
     "intelligence": 49.93581136467332
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.002666666666666706,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 46.94881578010102,
    "Emphasis on Accuracy 60:20:20": 39.21315403069989,
    "Emphasis on Cost 20:20:60": 50.502096788302346,
    "Emphasis on Speed 20:60:20": 53.77335547708169,
    "Intelligence only": 31.442161631528652
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Perplexity AI (perplexity-ai on Hugging Face)",
   "axes": {
    "calibration": 77.99716887881635,
    "cost": 78.96751116224428,
    "intelligence": 45.47596631214893,
    "speed": 91.46033110721706
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1388990410573218,
      "ece_score": 72.22019178853564,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 71.25725589802184,
      "score_before_time_limit": 72.22019178853564,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.21333333333333337,
      "brier": 0.1441461176011003,
      "ece": 0.09914083214795784,
      "n": 375,
      "score": 79.74425045803292,
      "score_before_time_limit": 80.17183357040844,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.8933333333333333,
      "interval90_width": 0.34911111111111115,
      "n": 375,
      "nrps": 0.10279383903889637,
      "score": 82.99000028039433,
      "score_before_time_limit": 83.65927447620396,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.11201033571851225
     }
    },
    "score": 77.99716887881635
   },
   "capability": 61.73656759548264,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 759,310 input + 0 output tokens over 1,485 decisions = 511.31986532 tokens/decision. GPU-time cost USD 0.00502438558860971/1,000; old list-price cost USD 0.015339595959596/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00502438558860971/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0387841
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005024385588609707
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "pplx-decider-flash-v1.2 (Perplexity, Qwen3.5-4B full fine-tune, marker readout, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-19",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (compute capability 12.0, driver 580.178.04), FIN-02 (Finland, EU), pay-as-you-go non-spot, USD 2.395/h, instance id 0c5da6de-cf37-4754-ba2f-718e990499d4. The author's own `Decider.decide()` from the pinned commit, served behind our own 135-line stdlib HTTP shim (sha256 911747c9ed275c6f5734e4f0bf8466741600ef94960d80641139b1779822c349) on 127.0.0.1:8034 inside a network namespace with no route and a proven cut, offline and credential-free, one request at a time, reference path (fast=False, never fp8). Timed over loopback on the VM. Speed was measured while the host VM was downloading other weights; the measured server remained in its network namespace with the cut proven.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 50.15338750065794,
    "I_sealed": 40.79854512363991,
    "base": 45.47596631214893,
    "excess": -0.4846843933376803,
    "gap": 9.35484237701803,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 67.09257921129485,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 58.47750865051904,
       "judge": 68.35860250494397,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 43.55693581780538,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 39.13043478260869,
       "judge": 19.999999999999996,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 39.81064747287361,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.609414282686686,
       "hard": 52.74275896247781,
       "judge": 37.96749037294228,
       "standard": 10.311776738655675
      }
     },
     "sealed|choice": {
      "cc": 58.79625662411246,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 80.94027954256671,
       "hard": 52.642534983231165,
       "judge": 52.39960822722822,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 29.46198834618804,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 44.827586206896555,
       "hard": 23.636363636363633,
       "judge": 16.92307692307693,
       "standard": 52.23880597014925
      }
     },
     "sealed|score": {
      "cc": 34.13739040061925,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 58.28165764661451,
       "hard": 31.500132808177707,
       "judge": 23.579999688936613,
       "standard": 43.17585803002867
      }
     }
    }
   },
   "jevbench_score": 56.65175338549098,
   "key": "pplx-decider-flash-v1.2",
   "last_measured_on": "2026-10-19",
   "licence": "Apache-2.0 (LICENSE + NOTICE in the repo and the HF metadata; base Qwen/Qwen3.5-4B Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.31",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "perplexity-ai/pplx-decider-flash-v1.2@6d771ad7b5ecff5e93b8fc7b90d4c5ccc341954d",
   "noul_decisive": {
    "acc_among_decisive": 0.8703071672354948,
    "decisive": 293,
    "decisive_rate": 0.7855227882037533,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.68376661171601,
     "cost": 64.42558238272366,
     "intelligence": 46.343042310166375,
     "speed": 91.76093007933393
    },
    "board_release": "v1.6.43",
    "capability": 62.51340446094119,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 48,
    "capability_rank": 73,
    "composite_ci95": {
     "A": [
      41.08146712316482,
      68.44059689877392
     ],
     "B": [
      37.078763157794235,
      64.12134021639524
     ],
     "C": [
      28.5287966133089,
      49.54196301366422
     ]
    },
    "jevbench_score": 56.60442934796988,
    "measured_in": "v1.6.31",
    "method": "v1.6",
    "open_board_rank": 35,
    "rank": 59,
    "ranks": {
     "A": 59,
     "B": 61,
     "C": 65,
     "capability": 115
    },
    "scores": {
     "A": 56.60442934796988,
     "B": 52.200826013455654,
     "C": 39.30863149164575
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23424214299975574,
     "p50_s_raw": 0.04212107149987787,
     "p95_s_adjusted": 0.284604628999773,
     "p95_s_raw": 0.06730231449988651
    },
    "views": {
     "Balanced 33:33:33": 53.694309763093415,
     "Emphasis on Accuracy 60:20:20": 47.12180219293807,
     "Emphasis on Cost 20:20:60": 54.3430713515502,
     "Emphasis on Speed 20:60:20": 61.543638587494364,
     "Intelligence only": 39.81196500845765
    }
   },
   "rank": 115,
   "ranked": true,
   "ranks": {
    "A": 115,
    "B": 117,
    "C": 120,
    "capability": 154
   },
   "repo": "https://huggingface.co/perplexity-ai/pplx-decider-flash-v1.2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 56.65175338549098,
    "B": 51.44603592123425,
    "C": 39.34149540659095
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.238672914999961,
    "p50_s_raw": 0.0443364574999805,
    "p95_s_adjusted": 0.2993393091003099,
    "p95_s_raw": 0.07466965455015497
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 23.890753145000755,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100,
        "score": -86.66107389833108
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 23.890753145000755
      },
      "calibration_gen": {
       "accuracy": 0.5544554455445545,
       "cc": 31.062896111775434,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": 19.999999999999996,
        "score": 33.63407539789257
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 31.062896111775434
      },
      "composite_gen": {
       "accuracy": 0.7821782178217822,
       "cc": 63.085637888757354,
       "cc_by_type": {
        "choice": 80.48780487804878,
        "noul": 55.55555555555556,
        "score": 53.2135532326677
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 63.085637888757354
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 73.17224732826755,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 59.516741984802614
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 73.17224732826755
      },
      "jaggedness": {
       "accuracy": 0.5444444444444444,
       "cc": 27.336682346573767,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": -2.7027027027026973,
        "score": 42.46058419863188
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 27.336682346573767
      },
      "llm_router": {
       "accuracy": 0.7603833865814696,
       "cc": 62.758982543003675,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 73.1958762886598,
        "score": 49.94943446441569
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 62.758982543003675
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -32.77757110043044,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -75,
        "score": -24.363641136342864
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6327272727272727,
       "cc": 40.176758046887215,
       "cc_by_type": {
        "choice": 58.9175089664167,
        "noul": 22.222222222222232,
        "score": 39.39054295202272
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 40.176758046887215
      },
      "negation_pair_gen": {
       "accuracy": 0.7558139534883721,
       "cc": 47.13079509036586,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 33.33333333333333,
        "score": 14.725718604430915
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 47.13079509036586
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 86.6287203707147,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 71.89817312415613
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 86.6287203707147
      },
      "usecase": {
       "accuracy": 0.5454545454545454,
       "cc": 29.798882678319305,
       "cc_by_type": {
        "choice": 50.98039215686277,
        "noul": 8.333333333333325,
        "score": 30.082922544761825
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 29.798882678319305
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -1.3047979839876547,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50,
        "score": -58.67629871386772
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 22.06955126716424,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": -25.65002515396042
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 22.06955126716424
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 74.45606493117315,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100,
        "score": 63.191203643076975
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 74.45606493117315
      },
      "el": {
       "accuracy": 0.4,
       "cc": 22.042381528754813,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.8329173981825
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 22.042381528754813
      },
      "en": {
       "accuracy": 0.6715899218071243,
       "cc": 46.770940309835645,
       "cc_by_type": {
        "choice": 62.65605740715879,
        "noul": 38.84892086330935,
        "score": 38.80784265903879
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 46.770940309835645
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 66.42316226760575,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 9.05741502467694
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.42316226760575
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 28.01144305470872,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 49.06165156849775
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 28.01144305470872
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 43.274381438995476,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 72.17608549345701
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 43.274381438995476
      },
      "hi": {
       "accuracy": 0.42857142857142855,
       "cc": -19.93619014374724,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -50,
        "score": -59.52924082230317
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 1,
       "cc": 85.83561523292173,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 57.50684569876518
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 85.83561523292173
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 23.825418841695438,
       "cc_by_type": {
        "choice": -0.07942811755361648,
        "noul": 60.00000000000001,
        "score": 11.555684642639918
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 23.825418841695438
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -3.856451682651605,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 32.875089396489635
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.03423249272396,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 97.76936414483853
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.03423249272396
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -15.253284155506398,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 6.188199481532752
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 53.750947209420126,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 56.48771255504834
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 53.750947209420126
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 65.2579740938948,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": 48.80868951798375
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 65.2579740938948
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 24.622498041966832,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 30.463238806751548
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 24.622498041966832
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 40.810477316435026,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100,
        "score": -34.09030718112971
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 40.810477316435026
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": 43.32203846875467,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": -5.517755561477933
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 43.32203846875467
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 7.235144418405379,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -19.537504597891207
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 7.235144418405379
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7026666666666667,
       "cc": 61.595131938996154,
       "cc_by_type": {
        "choice": 61.595131938996154
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 61.595131938996154
      },
      "noul": {
       "accuracy": 0.68,
       "cc": 36.00000000000001,
       "cc_by_type": {
        "noul": 36.00000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 36.00000000000001
      },
      "score": {
       "accuracy": 0.5653333333333334,
       "cc": 37.728820729894316,
       "cc_by_type": {
        "score": 37.728820729894316
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 37.728820729894316
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -32.3225310837819,
       "cc_by_type": {
        "choice": 20,
        "noul": -75,
        "score": -41.96759325134569
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 4.685368541529724,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": -22.520843995838714
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 4.685368541529724
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 51.559634286027205,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 53.45939066295966
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 51.559634286027205
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -32.66586536822346,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -16.12625107543062
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": 12.927739876171662,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 30.08756745460194
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 12.927739876171662
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -83.98969555652734,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": 0,
        "score": -276.49738855637446
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 32.191253901223796,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 1.8958905552119987
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 32.191253901223796
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 9.34672579166523,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -50,
        "score": 30.30498573736502
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 9.34672579166523
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 3.1383334542369234,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60,
        "score": 52.7483336960441
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 3.1383334542369234
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 35.158166165270266,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 23.53302811750958
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 35.158166165270266
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 43.18587719402969,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 11.111111111111116,
        "score": 30.07442744772214
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 43.18587719402969
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -11.668195633176785,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": -4.897060017809918
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 32.82004153950498,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": 19.999999999999996,
        "score": 36.0416985148681
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 32.82004153950498
      },
      "model_routing": {
       "accuracy": 0.8064516129032258,
       "cc": 66.92352968172027,
       "cc_by_type": {
        "choice": 78.96103896103895,
        "noul": 87.87878787878789,
        "score": 33.930762205334
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 66.92352968172027
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 9.844533432887033,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 52.56622985720236
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 9.844533432887033
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 47.70294210663885,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 43.20167683059438
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 47.70294210663885
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -5.190585022166457,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50,
        "score": -6.458454573888539
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 5.823091666909516,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 23.26637645000391
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 5.823091666909516
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 85.12622859667452,
       "cc_by_type": {
        "choice": 67.97153024911032,
        "noul": 100,
        "score": 87.40715554091322
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 85.12622859667452
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      73.05211676751716,
      81.0760443951826
     ],
     "I": [
      39.610873870051066,
      50.26724676027738
     ],
     "capability": [
      57.416919513219,
      65.18283792776047
     ],
     "composite": null,
     "seed": 59
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.99716887881635,
     "capability": 61.73656759548264,
     "intelligence": 45.47596631214893
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7026666666666667,
      "calibration": 71.25725589802184,
      "cc": 61.595131938996154,
      "cc_by_type": {
       "choice": 61.595131938996154
      },
      "cc_tiered": 62.94441791770365,
      "cc_tiered_open": 67.09257921129485,
      "cc_tiered_sealed": 58.79625662411246,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.595131938996154
     },
     "noul": {
      "accuracy": 0.68,
      "calibration": 79.74425045803292,
      "cc": 36.00000000000001,
      "cc_by_type": {
       "noul": 36.00000000000001
      },
      "cc_tiered": 36.50946208199671,
      "cc_tiered_open": 43.55693581780538,
      "cc_tiered_sealed": 29.46198834618804,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 36.00000000000001
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 82.99000028039433,
      "cc": 37.728820729894316,
      "cc_by_type": {
       "score": 37.728820729894316
      },
      "cc_tiered": 36.97401893674643,
      "cc_tiered_open": 39.81064747287361,
      "cc_tiered_sealed": 34.13739040061925,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 37.728820729894316
     }
    },
    "raw_on_A": {
     "calibration": 77.72244200758998,
     "capability": 62.348124101385025,
     "intelligence": 46.97380619518008
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 54.43851256329439,
    "Emphasis on Accuracy 60:20:20": 46.17963012282767,
    "Emphasis on Cost 20:20:60": 58.32624951319447,
    "Emphasis on Speed 20:60:20": 61.31758035676897,
    "Intelligence only": 37.61887464166974
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Perplexity",
   "axes": {
    "calibration": 89.82871864473867,
    "cost": 57.33119184627563,
    "intelligence": 75.59095372376527,
    "speed": 86.35967790128073
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06981468503472926,
      "ece_score": 86.03706299305415,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 85.9223469090634,
      "score_before_time_limit": 86.03706299305415,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.0889487870619946,
      "brier": 0.062186325211303205,
      "ece": 0.051085406690107325,
      "n": 371,
      "score": 89.78291866197854,
      "score_before_time_limit": 89.78291866197854,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9566395663956639,
      "interval90_width": 0.165085817524842,
      "n": 369,
      "nrps": 0.04050151348288236,
      "score": 93.78089036317405,
      "score_before_time_limit": 93.78089036317405,
      "time_limit_failed_share": 0,
      "top_ece": 0.04194033962681827
     }
    },
    "score": 89.82871864473867
   },
   "capability": 82.70983618425197,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11830.7366892 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=trunk-equivalent). The row's own scored cost basis: 764,265 input + 0 output tokens over 1,477 decisions = 517.444143534 tokens/decision. GPU-time cost USD 0.0264419576381443/1,000; old list-price cost USD 0.21732654028436/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0264419576381443/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.21399420000000002
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.026441957638144272
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Perplexity Decider v1.1 27B (Qwen3.8-27B, noncausal decision readout)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 78.11444356279019,
    "I_sealed": 73.06746388474036,
    "base": 75.59095372376527,
    "excess": 2.473592035611579,
    "gap": 5.046979678049823,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 86.99770517622916,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 84.90091223655237,
       "judge": 88.13447593935398,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 77.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 50,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 70.26833082615104,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.42506419505849,
       "hard": 75.99276310558226,
       "judge": 80.94030641691248,
       "standard": 46.23313619669274
      }
     },
     "sealed|choice": {
      "cc": 80.03085922219567,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 86.6972138053359,
       "hard": 71.49321266968325,
       "judge": 86.96461824953444,
       "standard": 83.37233649464221
      }
     },
     "sealed|noul": {
      "cc": 66.96797052113396,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 53.398058252427184,
       "judge": 71.05263157894737,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 72.20356191089144,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 73.73143188188055,
       "hard": 66.84626737629874,
       "judge": 80.0022436490447,
       "standard": 70.4561933873524
      }
     }
    }
   },
   "jevbench_score": 74.92968283438655,
   "key": "pplx-decider-v1-1-27b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "perplexity-ai/pplx-decider-v1.1-27b@3b45dead91dfa6d95aad6b95764a606fab2bf7a6",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9497041420118343,
    "decisive_rate": 0.9110512129380054,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.86695733940225,
     "cost": 29.886617009940878,
     "intelligence": 75.71677945512734,
     "speed": 86.79279341461623
    },
    "board_release": "v1.6.43",
    "capability": 82.79186839726479,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.265432194933428,
      20.803744956997086
     ],
     "B": [
      21.111865782219617,
      21.95821416195702
     ],
     "C": [
      20.265432194933428,
      20.803744956997086
     ]
    },
    "jevbench_score": 20.617642759667973,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 93,
    "rank": 127,
    "ranks": {
     "A": 127,
     "B": 117,
     "C": 109,
     "capability": 12
    },
    "scores": {
     "A": 20.617642759667973,
     "B": 21.647466573268815,
     "C": 20.617642759667973
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.39472174100810664,
     "p50_s_raw": 0.12236087050405331,
     "p95_s_adjusted": 0.5301876544195693,
     "p95_s_raw": 0.19009382720978463
    },
    "views": {
     "Balanced 33:33:33": 18.420307065234873,
     "Emphasis on Accuracy 60:20:20": 21.115368174311886,
     "Emphasis on Cost 20:20:60": 14.279000034452622,
     "Emphasis on Speed 20:60:20": 21.991580334127057,
     "Intelligence only": 27.052390084345777
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 20,
   "ranked": true,
   "ranks": {
    "A": 20,
    "B": 12,
    "C": 18,
    "capability": 14
   },
   "repo": "https://huggingface.co/perplexity-ai/pplx-decider-v1.1-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.92968283438655,
    "B": 75.06100982069017,
    "C": 74.92968283438655
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4121504910348449,
    "p50_s_raw": 0.13107524551742245,
    "p95_s_adjusted": 0.5610174738016211,
    "p95_s_raw": 0.2055087369008106
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.8241744764109,
      91.28720890209534
     ],
     "I": [
      70.85113772743928,
      78.95089291005449
     ],
     "capability": [
      79.1028452853472,
      84.63982686991523
     ],
     "composite": null,
     "seed": 95
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.82871864473867,
     "capability": 82.70983618425197,
     "intelligence": 75.59095372376527
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8546666666666667,
      "calibration": 85.9223469090634,
      "cc": 81.41394238511599,
      "cc_by_type": {
       "choice": 81.41394238511599
      },
      "cc_tiered": 83.51428219921242,
      "cc_tiered_open": 86.99770517622916,
      "cc_tiered_sealed": 80.03085922219567,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.41394238511599
     },
     "noul": {
      "accuracy": 0.856,
      "calibration": 89.78291866197854,
      "cc": 71.2,
      "cc_by_type": {
       "noul": 71.2
      },
      "cc_tiered": 72.02263260356216,
      "cc_tiered_open": 77.07729468599035,
      "cc_tiered_sealed": 66.96797052113396,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 71.2
     },
     "score": {
      "accuracy": 0.8133333333333334,
      "calibration": 93.78089036317405,
      "cc": 71.25760707133534,
      "cc_by_type": {
       "score": 71.25760707133534
      },
      "cc_tiered": 71.23594636852124,
      "cc_tiered_open": 70.26833082615104,
      "cc_tiered_sealed": 72.20356191089144,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 71.25760707133534
     }
    },
    "raw_on_A": {
     "calibration": 89.31143519070639,
     "capability": 82.8788987724678,
     "intelligence": 76.44636235422921
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01866666666666672,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 71.00409155813412,
    "Emphasis on Accuracy 60:20:20": 72.7703752986783,
    "Emphasis on Cost 20:20:60": 64.8204840066085,
    "Emphasis on Speed 20:60:20": 76.4408614820513,
    "Intelligence only": 75.59095372376527
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Perplexity AI (perplexity-ai on Hugging Face)",
   "axes": {
    "calibration": 89.96214607141948,
    "cost": 57.41150540159515,
    "intelligence": 71.71777888018025,
    "speed": 85.96624125269479
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04781048796115123,
      "ece_score": 90.43790240776976,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 89.23206370899949,
      "score_before_time_limit": 90.43790240776976,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.16266666666666663,
      "brier": 0.06680390854205732,
      "ece": 0.05588531133627415,
      "n": 375,
      "score": 88.34921539817053,
      "score_before_time_limit": 88.82293773274516,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.976,
      "interval90_width": 0.21555555555555558,
      "n": 375,
      "nrps": 0.045355161612738365,
      "score": 92.30515910708841,
      "score_before_time_limit": 93.04955555150042,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.046826863678626575
     }
    },
    "score": 89.96214607141948
   },
   "capability": 80.83996247579987,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 759,310 input + 0 output tokens over 1,485 decisions = 511.31986532 tokens/decision. GPU-time cost USD 0.0262794630059689/1,000; old list-price cost USD 0.214754343434343/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0262794630059689/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.5429773999999999
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02627946300596891
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "pplx-decider-v1.2 (Perplexity, Qwen3.8-27B full fine-tune, marker readout, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-20",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA H200 141 GB (compute capability 9.0, driver 580.178.04), FIN-02 (Finland, EU), pay-as-you-go non-spot, instance type 2H200.141S.88V at USD 11.32/h with the second GPU unbound from the driver before anything ran (nvidia-smi and torch both saw exactly one GPU; single-GPU, no tensor parallelism). The author's own `Decider.decide()` from the pinned commit, served behind our own 135-line stdlib HTTP shim (sha256 911747c9ed275c6f5734e4f0bf8466741600ef94960d80641139b1779822c349) on 127.0.0.1:8035 inside a network namespace with no route and a proven cut, offline and credential-free, one request at a time, reference path (fast=False, never fp8). Timed over loopback on the VM. Measured 20 Oct 2026.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 77.10920374861246,
    "I_sealed": 66.32635401174804,
    "base": 71.71777888018025,
    "excess": 0.9433229665087097,
    "gap": 10.78284973686442,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 86.74318374569839,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 96.044825313118,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 73.42650103519671,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 91.30434782608697,
       "judge": 50,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 71.15792646494232,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 75.33980083493176,
       "hard": 77.46041640319304,
       "judge": 72.97602843031399,
       "standard": 53.734856455388666
      }
     },
     "sealed|choice": {
      "cc": 80.44759269360293,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 74.49982652943217,
       "judge": 81.48873653281098,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 54.272083037909084,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 45.45454545454546,
       "judge": 50.76923076923077,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 64.25938630373211,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 84.77887937684766,
       "hard": 61.98686138973364,
       "judge": 57.77258889166228,
       "standard": 68.27488571327603
      }
     }
    }
   },
   "jevbench_score": 73.92360529540504,
   "key": "pplx-decider-v1.2",
   "last_measured_on": "2026-10-20",
   "licence": "Apache-2.0 (LICENSE + NOTICE in the repo and the HF metadata; base Qwen/Qwen3.8-27B Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.33",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "perplexity-ai/pplx-decider-v1.2@225c913ddc8761a8264f04f64548e1f97e443eb3",
   "noul_decisive": {
    "acc_among_decisive": 0.9648562300319489,
    "decisive": 313,
    "decisive_rate": 0.839142091152815,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.77013189733844,
     "cost": 30.0417413123765,
     "intelligence": 72.71358625871065,
     "speed": 86.32357007699105
    },
    "board_release": "v1.6.43",
    "capability": 81.74185907802455,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.385257230642424,
      20.922079167304812
     ],
     "B": [
      21.116487641783483,
      21.96657076729541
     ],
     "C": [
      20.385257230642424,
      20.922079167304812
     ]
    },
    "jevbench_score": 20.73512390064423,
    "measured_in": "v1.6.33",
    "method": "v1.6",
    "open_board_rank": 91,
    "rank": 125,
    "ranks": {
     "A": 125,
     "B": 118,
     "C": 107,
     "capability": 17
    },
    "scores": {
     "A": 20.73512390064423,
     "B": 21.644559304900415,
     "C": 20.73512390064423
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.43327262599992766,
     "p50_s_raw": 0.14163631299996382,
     "p95_s_adjusted": 0.5381232308001244,
     "p95_s_raw": 0.1940616154000622
    },
    "views": {
     "Balanced 33:33:33": 18.473812614761474,
     "Emphasis on Accuracy 60:20:20": 20.9570557390737,
     "Emphasis on Cost 20:20:60": 14.41726658567706,
     "Emphasis on Speed 20:60:20": 22.068170846951432,
     "Intelligence only": 26.249785582202126
    }
   },
   "rank": 34,
   "ranked": true,
   "ranks": {
    "A": 34,
    "B": 25,
    "C": 29,
    "capability": 31
   },
   "repo": "https://huggingface.co/perplexity-ai/pplx-decider-v1.2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.92360529540504,
    "B": 73.47165122883665,
    "C": 73.92360529540504
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.44363213599992835,
    "p50_s_raw": 0.14681606799996416,
    "p95_s_adjusted": 0.5706277350999812,
    "p95_s_raw": 0.2103138675499906
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 73.07521976445251,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 27.55899262669086
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 73.07521976445251
      },
      "calibration_gen": {
       "accuracy": 0.7029702970297029,
       "cc": 57.398871595668176,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": 52,
        "score": 64.73532104363233
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 57.398871595668176
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 86.73438142756389,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 85.18518518518519,
        "score": 78.26999161783166
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 86.73438142756389
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 84.4704120616969,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 93.41123618509071
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.4704120616969
      },
      "jaggedness": {
       "accuracy": 0.7444444444444445,
       "cc": 59.604085228758656,
       "cc_by_type": {
        "choice": 65.35129932627527,
        "noul": 35.13513513513513,
        "score": 78.32582122486556
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 59.604085228758656
      },
      "llm_router": {
       "accuracy": 0.9361022364217252,
       "cc": 85.56210896843349,
       "cc_by_type": {
        "choice": 95.24522321035484,
        "noul": 95.8762886597938,
        "score": 65.56481503515184
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 85.56210896843349
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5185185185185185,
       "cc": 20.88632537167448,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -25,
        "score": 49.51464621811623
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 20.88632537167448
      },
      "multilingual": {
       "accuracy": 0.7781818181818182,
       "cc": 62.04880430430405,
       "cc_by_type": {
        "choice": 81.41506358004565,
        "noul": 38.888888888888886,
        "score": 65.84246044397761
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 62.04880430430405
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 80.71787667169285,
       "cc_by_type": {
        "choice": 100,
        "noul": 73.33333333333334,
        "score": 68.82029668174519
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 80.71787667169285
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 79.18647345233632,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 52.574435372023984
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 79.18647345233632
      },
      "usecase": {
       "accuracy": 0.75,
       "cc": 59.529698223943285,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 37.5,
        "score": 64.37340839732005
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 59.529698223943285
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 48.013941666516395,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": 32.13706309478728
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 48.013941666516395
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 53.588644483199374,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 27.432600116264794
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 53.588644483199374
      },
      "de": {
       "accuracy": 0.875,
       "cc": 87.0741245994998,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 85.11617910823391
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.0741245994998
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 61.07592115708842,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 0,
        "score": 99.9875400075781
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 61.07592115708842
      },
      "en": {
       "accuracy": 0.8349261511728931,
       "cc": 72.82231693818183,
       "cc_by_type": {
        "choice": 82.64605020685615,
        "noul": 66.90647482014388,
        "score": 68.91442578754545
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 72.82231693818183
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 77.50244222824413,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 42.295254906592106
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 77.50244222824413
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 65.96563885246287,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 97.35046846995695
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.96563885246287
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 90.44568219340927,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 92.51351716846314
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 90.44568219340927
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 42.92298599748088,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 25,
        "score": 37.28851106506833
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 42.92298599748088
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 73.01823353533898,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 33.08431868863578
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 73.01823353533898
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 34.74270854767963,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 0,
        "score": 18.525186802689397
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 34.74270854767963
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 20.91054015944991,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 62.73162047834971
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 20.91054015944991
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.64112914545721,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.5900541030383
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.64112914545721
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -25.467887068329684,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": -50.429635230963086
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 77.98547426709224,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80,
        "score": 69.19129372806471
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 77.98547426709224
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 77.04784638664607,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 62.964399501717814
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 77.04784638664607
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 53.66153195458858,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 52.04842565099979
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 53.66153195458858
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 84.37759555821967,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 53.132786674659
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 84.37759555821967
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.70355972828668,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 95.36874370098906
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.70355972828668
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 49.13494079901958,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 14.071489063725428
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 49.13494079901958
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8613333333333333,
       "cc": 82.08920951415067,
       "cc_by_type": {
        "choice": 82.08920951415067
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 82.08920951415067
      },
      "noul": {
       "accuracy": 0.8053333333333333,
       "cc": 61.06666666666667,
       "cc_by_type": {
        "noul": 61.06666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 61.06666666666667
      },
      "score": {
       "accuracy": 0.7546666666666667,
       "cc": 67.14016711842746,
       "cc_by_type": {
        "score": 67.14016711842746
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 67.14016711842746
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 10.59424018776719,
       "cc_by_type": {
        "choice": 68,
        "noul": -50,
        "score": 13.782720563301565
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 10.59424018776719
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 87.58326240675524,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 62.74978722026573
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 87.58326240675524
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 38.482668211129294,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 14.228492438265938
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 38.482668211129294
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -10.890476546823086,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -95.82932437731135
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 38.4736130134271,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 77.73967961999143
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 38.4736130134271
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 33.292688730256714,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": -20.876650790361918
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 33.292688730256714
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 38.126345773148394,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 58.076516311041814
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 38.126345773148394
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 60.05026660124205,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 80.15079980372617
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 60.05026660124205
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 15.085726936490197,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60,
        "score": 46.92384747613726
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 15.085726936490197
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 74.46899896013086,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 69.58829595248395
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 74.46899896013086
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 63.797278280314096,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 47.464186262130944
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 63.797278280314096
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": 11.921579031604551,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 31.463661825996457
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 11.921579031604551
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 59.44426920442447,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 69.84912239254406
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.44426920442447
      },
      "model_routing": {
       "accuracy": 0.9170506912442397,
       "cc": 82.17204853771132,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 96.96969696969697,
        "score": 54.221773318761656
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 82.17204853771132
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 18.622337771983307,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 78.89964287449116
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 18.622337771983307
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 56.74843643361415,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 70.33815981152026
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 56.74843643361415
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 31.118849344180674,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": 22.913198278847435
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 31.118849344180674
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 27.996149618548205,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 75.29279668173157
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 27.996149618548205
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.85528914958637,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.56586744875911
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.85528914958637
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.78487270534086,
      91.02037761468912
     ],
     "I": [
      67.3887930401396,
      75.29632276772642
     ],
     "capability": [
      77.17563485429892,
      82.6025660243471
     ],
     "composite": null,
     "seed": 60
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.96214607141948,
     "capability": 80.83996247579987,
     "intelligence": 71.71777888018025
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8613333333333333,
      "calibration": 89.23206370899949,
      "cc": 82.08920951415067,
      "cc_by_type": {
       "choice": 82.08920951415067
      },
      "cc_tiered": 83.59538821965066,
      "cc_tiered_open": 86.74318374569839,
      "cc_tiered_sealed": 80.44759269360293,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 82.08920951415067
     },
     "noul": {
      "accuracy": 0.8053333333333333,
      "calibration": 88.34921539817053,
      "cc": 61.06666666666667,
      "cc_by_type": {
       "noul": 61.06666666666667
      },
      "cc_tiered": 63.849292036552896,
      "cc_tiered_open": 73.42650103519671,
      "cc_tiered_sealed": 54.272083037909084,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 61.06666666666667
     },
     "score": {
      "accuracy": 0.7546666666666667,
      "calibration": 92.30515910708841,
      "cc": 67.14016711842746,
      "cc_by_type": {
       "score": 67.14016711842746
      },
      "cc_tiered": 67.70865638433722,
      "cc_tiered_open": 71.15792646494232,
      "cc_tiered_sealed": 64.25938630373211,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 67.14016711842746
     }
    },
    "raw_on_A": {
     "calibration": 88.31082903662588,
     "capability": 79.78253309681389,
     "intelligence": 71.25423715700191
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.77696948195734,
    "Emphasis on Accuracy 60:20:20": 70.54055008751058,
    "Emphasis on Cost 20:20:60": 64.24228982681217,
    "Emphasis on Speed 20:60:20": 75.46136125100679,
    "Intelligence only": 71.71777888018025
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Olaverse (Hugging Face olaverse; benchmarkheaven.com/submit ref 6195A889)",
   "axes": {
    "calibration": 83.64479179791499,
    "cost": null,
    "intelligence": 36.57669575617585,
    "speed": 87.71257539670552
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08471827409306511,
      "ece_score": 83.05634518138699,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 83.05634518138699,
      "score_before_time_limit": 83.05634518138699,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3699731903485255,
      "brier": 0.14497102452798155,
      "ece": 0.08867916523731106,
      "n": 373,
      "score": 82.26416695253779,
      "score_before_time_limit": 82.26416695253779,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9247311827956989,
      "interval90_width": 0.39471326164874576,
      "n": 372,
      "nrps": 0.10271997153867293,
      "score": 85.61386325982022,
      "score_before_time_limit": 85.61386325982022,
      "time_limit_failed_share": 0,
      "top_ece": 0.09250138163246135
     }
    },
    "score": 83.64479179791499
   },
   "capability": 60.11074377704542,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 6202ef78..., the module in force at scoring time; the later re-freezes only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base (the module RAISES 'base model lacks an explicit market-listing decision', for every spelling tried); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void - the same treatment as the live jevora-9b-v1 and kev-9b-v2 rows on that same declared base. The tokens needed for a later reprice are measured and on file: the author's server reports NO usage block at all, so the input tokens were counted OFFLINE with the author's own pinned tokenizer and their own unchanged encode() on the exact prompts their server builds from the runner's request bodies (the pricing update, as accepted for the sibling row): 700,492 input tokens over the 1,485 answered items, 0 output (nothing is generated). That count is byte-for-byte identical, item by item, to the count accepted for purplemist-mini-1.0 - same rendered-prompt-ids sha256 ad0810696b7e80eba87192caa70f57c19cdbee103e37f3d9cd163ef5df434c86 - which it must be, because the rendered prompt depends only on the tokenizer, encode(), the two flags and the request body, all byte-identical between the two rows, and not on the weights.",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "PurpleMIST-Flash-1.0 (Olaverse)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-17",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod, 1x NVIDIA RTX 6000 Ada Generation 48 GB (sm_89, driver 595.84), Athens, Greece (EU), secure tier (not preemptible), USD 0.80/h, 05:26-05:53Z on 17 Oct 2026 = USD 0.35; removed by exact id and verified gone four ways. Speed is as measured on that card and is NOT strictly comparable to the H100/RTX 5090 rows. Isolation: the author's server ran inside a network namespace with loopback only, with the cut PROVEN before any sealed item was dispatched (no interfaces but lo, no routes, outbound by name and by IP both failing) and proven again immediately before the first sealed request; credential-free environment (env -i, checked), weights from a locally verified offline snapshot of the pinned revision (11 files, 15,927,059,974 bytes, model.safetensors sha256 2dba83efd90dbe2fcf767eaa004f48789029cff3282f7c6ae8a66eca7dc26202), sealed input deleted on the pod at 05:51:55Z and the server's process group killed by exact pid. One request at a time. Transfer deviation: the Hugging Face Xet path stalled on this host (its own log reported 'connection struggling' at a success ratio of 1.000 while the link was taking 25 MB/s), so the snapshot was fetched over plain HTTPS with HF_HUB_DISABLE_XET=1; what is verified did not change.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.963930901767135,
    "I_sealed": 31.18946061058456,
    "base": 36.57669575617585,
    "excess": 0.934943520826863,
    "gap": 10.774470291182574,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 63.66615272089087,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 60.36489462094998,
       "judge": 68.35860250494397,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 17.12008281573499,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 4.347826086956519,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 45.105557168675546,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.05582660274707,
       "hard": 49.94314086701773,
       "judge": 35.26365964380409,
       "standard": 45.718101342262564
      }
     },
     "sealed|choice": {
      "cc": 57.693838803821905,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 80.94027954256671,
       "hard": 48.47924135538336,
       "judge": 56.80705190989228,
       "standard": 65.82999367222105
      }
     },
     "sealed|noul": {
      "cc": 4.470946449330393,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 31.034482758620683,
       "hard": -3.6363636363636376,
       "judge": -1.538461538461533,
       "standard": 16.417910447761198
      }
     },
     "sealed|score": {
      "cc": 31.403596578601384,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 61.046601730525076,
       "hard": 25.44469977308672,
       "judge": 23.561584549581017,
       "standard": 40.2629056571994
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "purplemist-flash-1.0",
   "last_measured_on": "2026-10-17",
   "licence": "Apache-2.0 (model card and HF metadata), public, not gated",
   "listing": "listed",
   "measured_in": "v1.6.28",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "olaverse/PurpleMIST-Flash-1.0@a48fe0206ee0d8a3f4458a3581468c24a7365ac5",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base Qwen/Qwen3.5-9B-Base and raises rather than estimating; one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.8851063829787233,
    "decisive": 235,
    "decisive_rate": 0.6300268096514745,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.64479179791499,
     "cost": null,
     "intelligence": 36.57669575617585,
     "speed": 88.25066979413353
    },
    "board_release": "v1.6.43",
    "capability": 60.11074377704542,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.28",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.31827442100111514,
     "p50_s_raw": 0.08413721050055756,
     "p95_s_adjusted": 0.4700361753995821,
     "p95_s_raw": 0.16001808769979106
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/olaverse/PurpleMIST-Flash-1.0",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.33086025799857455,
    "p50_s_raw": 0.09043012899928726,
    "p95_s_adjusted": 0.511797141898569,
    "p95_s_raw": 0.18089857094928447
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 70.39546327561969,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 44.51972316019237
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 70.39546327561969
      },
      "calibration_gen": {
       "accuracy": 0.49504950495049505,
       "cc": 17.209450111800084,
       "cc_by_type": {
        "choice": 42.73594909862143,
        "noul": -12,
        "score": 20.892401236778824
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 17.209450111800084
      },
      "composite_gen": {
       "accuracy": 0.6138613861386139,
       "cc": 35.56560256341613,
       "cc_by_type": {
        "choice": 57.72357723577236,
        "noul": 18.518518518518512,
        "score": 30.454711935957533
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 35.56560256341613
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 72.27956011126733,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 56.838680333801975
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 72.27956011126733
      },
      "jaggedness": {
       "accuracy": 0.46111111111111114,
       "cc": 2.3307448578111356,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": -56.75675675675676,
        "score": 21.496825786398045
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 2.3307448578111356
      },
      "llm_router": {
       "accuracy": 0.7380191693290735,
       "cc": 56.233400214405066,
       "cc_by_type": {
        "choice": 70.67887646385489,
        "noul": 46.39175257731958,
        "score": 51.62957160204073
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 56.233400214405066
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -17.44240188691204,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75,
        "score": -3.1004015370247773
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6363636363636364,
       "cc": 38.55335313614513,
       "cc_by_type": {
        "choice": 62.83012716009129,
        "noul": 2.777777777777768,
        "score": 50.052154470566315
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 38.55335313614513
      },
      "negation_pair_gen": {
       "accuracy": 0.5348837209302325,
       "cc": 15.35444594072437,
       "cc_by_type": {
        "choice": 53.333333333333336,
        "noul": -19.999999999999996,
        "score": 12.730004488839764
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 15.35444594072437
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 78.93817096043851,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 100,
        "score": 72.85054891735155
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 78.93817096043851
      },
      "usecase": {
       "accuracy": 0.5863636363636363,
       "cc": 32.92209243846133,
       "cc_by_type": {
        "choice": 55.88235294117649,
        "noul": 8.333333333333325,
        "score": 34.550591040874146
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 32.92209243846133
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -1.182611711477912,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0,
        "score": -27.357358943957543
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 52.0471958671179,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 22.808254268020377
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 52.0471958671179
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 77.26496186483496,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 80.68869090423941
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 77.26496186483496
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 16.45188091170487,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -66.66666666666667,
        "score": 99.82119208334552
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 16.45188091170487
      },
      "en": {
       "accuracy": 0.6168549087749783,
       "cc": 34.58332742333039,
       "cc_by_type": {
        "choice": 58.48232264678242,
        "noul": 11.51079136690647,
        "score": 33.75686825630226
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 34.58332742333039
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 57.29713259637261,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75,
        "score": 6.679326010977515
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 57.29713259637261
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 29.410935130872797,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 87.13990921775499
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 29.410935130872797
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 53.565431529597426,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 81.87276517702757
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 53.565431529597426
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 13.66047649462213,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25,
        "score": -0.4990174435079098
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 13.66047649462213
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 56.84148427361694,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": -1.416311013911442
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 56.84148427361694
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 26.796685072403772,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 17.578299855813384
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 26.796685072403772
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -5.669662434756812,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 27.435457140174012
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.54577863099985,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.63733589299954
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.54577863099985
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 7.617863381918501,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": 74.80164209380746
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 7.617863381918501
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 47.5434181308745,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 37.86512531941148
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 47.5434181308745
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 33.36042327927317,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": -19.999999999999996,
        "score": 51.902130179599126
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 33.36042327927317
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 11.505633172783869,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 56.64455909281969
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 11.505633172783869
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": 0.08992728763732316,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -50,
        "score": -6.251957267522812
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0.08992728763732316
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.95162904215476,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 92.37101615872234
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.95162904215476
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 19.93758488161902,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": -73.52057868847626
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 19.93758488161902
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.688,
       "cc": 59.70072140683901,
       "cc_by_type": {
        "choice": 59.70072140683901
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 59.70072140683901
      },
      "noul": {
       "accuracy": 0.5546666666666666,
       "cc": 10.933333333333328,
       "cc_by_type": {
        "noul": 10.933333333333328
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 10.933333333333328
      },
      "score": {
       "accuracy": 0.5573333333333333,
       "cc": 35.98058722164548,
       "cc_by_type": {
        "score": 35.98058722164548
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 35.98058722164548
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": -8.023171759301837,
       "cc_by_type": {
        "choice": 52,
        "noul": -25,
        "score": -51.06951527790551
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 18.05749693437015,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 33.33333333333333,
        "score": -4.6266810395396485
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 18.05749693437015
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 28.239231994773096,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": 33.49818378919734
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 28.239231994773096
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -13.419745719245478,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": 29.916201438754797
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 19.48037661141311,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 49.74547766032629
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 19.48037661141311
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -47.63928647397137,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100,
        "score": -54.8675449565053
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 40.90857018259701,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 28.047839399331654
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 40.90857018259701
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -9.471877723757162,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100,
        "score": 34.30213686357171
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 34.06036190912817,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 63.847752394051184
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 34.06036190912817
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 16.292238984459793,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 0,
        "score": 38.81247712468559
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 16.292238984459793
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 38.280305827415546,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 37.57993557010195
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 38.280305827415546
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -32.35528164753511,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100,
        "score": -0.2916513942182286
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 35.00919149670052,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 19.999999999999996,
        "score": 19.576518827913468
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 35.00919149670052
      },
      "model_routing": {
       "accuracy": 0.783410138248848,
       "cc": 58.2495546137415,
       "cc_by_type": {
        "choice": 87.14285714285714,
        "noul": 51.515151515151516,
        "score": 36.090655183215844
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 58.2495546137415
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 3.368242149671545,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100,
        "score": 67.68630034536781
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 3.368242149671545
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 44.13000314873262,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 33.33333333333333,
        "score": 49.19595187888124
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 44.13000314873262
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 21.425366213581725,
       "cc_by_type": {
        "choice": 100,
        "noul": -50,
        "score": 14.276098640745172
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 21.425366213581725
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 15.648163302975552,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 52.74159135820202
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 15.648163302975552
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 85.73439203830024,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100,
        "score": 78.55548928216051
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 85.73439203830024
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.09728138937768,
      85.59062383959186
     ],
     "I": [
      31.13234243642542,
      41.14289724655533
     ],
     "capability": [
      55.33910988485174,
      62.724058487049085
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 61
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.64479179791499,
     "capability": 60.11074377704542,
     "intelligence": 36.57669575617585
    },
    "per_type": {
     "choice": {
      "accuracy": 0.688,
      "calibration": 83.05634518138699,
      "cc": 59.70072140683901,
      "cc_by_type": {
       "choice": 59.70072140683901
      },
      "cc_tiered": 60.679995762356384,
      "cc_tiered_open": 63.66615272089087,
      "cc_tiered_sealed": 57.693838803821905,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.70072140683901
     },
     "noul": {
      "accuracy": 0.5546666666666666,
      "calibration": 82.26416695253779,
      "cc": 10.933333333333328,
      "cc_by_type": {
       "noul": 10.933333333333328
      },
      "cc_tiered": 10.79551463253269,
      "cc_tiered_open": 17.12008281573499,
      "cc_tiered_sealed": 4.470946449330393,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 10.933333333333328
     },
     "score": {
      "accuracy": 0.5573333333333333,
      "calibration": 85.61386325982022,
      "cc": 35.98058722164548,
      "cc_by_type": {
       "score": 35.98058722164548
      },
      "cc_tiered": 38.25457687363846,
      "cc_tiered_open": 45.105557168675546,
      "cc_tiered_sealed": 31.403596578601384,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 35.98058722164548
     }
    },
    "raw_on_A": {
     "calibration": 86.19021939188609,
     "capability": 62.4812521197941,
     "intelligence": 38.77228484770211
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Olaverse (contact@olaverse.co.uk)",
   "axes": {
    "calibration": 77.68611335416006,
    "cost": 86.34644068316165,
    "intelligence": 8.377579633895749,
    "speed": 88.54013979409211
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1300396062800142,
      "ece_score": 73.99207874399715,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 73.99207874399715,
      "score_before_time_limit": 73.99207874399715,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.9142091152815013,
      "brier": 0.22633030087611059,
      "ece": 0.12555655399612065,
      "n": 373,
      "score": 74.88868920077587,
      "score_before_time_limit": 74.88868920077587,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9354838709677419,
      "interval90_width": 0.7486559139784947,
      "n": 372,
      "nrps": 0.16595993804336154,
      "score": 84.17757211770714,
      "score_before_time_limit": 84.17757211770714,
      "time_limit_failed_share": 0,
      "top_ece": 0.07524430980124781
     }
    },
    "score": 77.68611335416006
   },
   "capability": 43.0318464940279,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 2B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 700,492 input + 0 output tokens over 1,485 decisions = 471.711784512 tokens/decision. GPU-time cost USD 0.00285179723691946/1,000; old list-price cost USD 0.00943423569023569/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00285179723691946/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.009339893333333333
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0028517972369194594
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "PurpleMIST-Mini-1.0 (Olaverse)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Verda RTX 6000 Ada (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-02, one dedicated GPU instance, network none (proven), evaluator-controlled self-hosted deployment. The author's own serve.py and purplemist.py, unchanged, at their declared GPU configuration: `python3 serve.py --model <pinned snapshot> --device cuda --max-len 65536 --no-truncate --host 127.0.0.1 --port <p>` (fp32 loaded automatically as the model's inference_dtype), with the author's server WARM-UP ON (the earlier CPU run used our --no-warmup, which is forbidden by the gate and absent from the launched argv). No deviation from the author's declared configuration other than the hardware. With --no-truncate the author's code refuses rather than truncates a prompt over 65,536 tokens, so the draw's 15 longest items are honest HTTP 422 refusals scored as wrong (the same 15 as in the CPU row). The author's server reports no token usage, so input tokens were counted offline with the author's pinned tokenizer (700,492 over the 1,485 answered items; same count as the CPU row). Speed is a dedicated-GPU figure, measured one request at a time; the standard self-hosted x2 + 0.15 s adjustment applies and raw p50/p95 are kept. RE-MEASURE ON THE SAME ITEMS: this GPU row REPLACES the earlier loaded-CPU row in full (scores AND times); the GPU times are not combined with the CPU scores. The same 1,500 items of draw v1.6-regular-20261012 (input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b), the same pinned weights. Pod: Verda 1RTX6000ADA.10V (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-02, torn down after the run (22 Oct 2026). The scoring run had no network: container NetworkMode none, only the loopback interface, cut proven by three failed outbound probes before the first sealed byte; read-only root, all capabilities dropped. The earlier CPU measurement of this row was its first use of the weights on this draw; this is the second and last.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 10.02427738947358,
    "I_sealed": 6.730881878317918,
    "base": 8.377579633895749,
    "excess": -6.546131259200049,
    "gap": 3.293395511155662,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 21.510322262497397,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.94594594594595,
       "hard": 1.8559295375904445,
       "judge": 32.76203032300594,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -91.30434782608697,
       "judge": -100,
       "standard": -100
      }
     },
     "open|score": {
      "cc": 8.562509905923346,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.11955505993331,
       "hard": -13.404645221049183,
       "judge": 24.543976136562172,
       "standard": 4.746098236905183
      }
     },
     "sealed|choice": {
      "cc": 20.192645634953756,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 19.94917407878019,
       "hard": 17.774950850005773,
       "judge": 21.547502448579845,
       "standard": 23.117485762497374
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -62.06896551724138,
       "hard": -92.72727272727272,
       "judge": -87.6923076923077,
       "standard": -85.07462686567165
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 9.068912569956279,
       "hard": -10.76273169840718,
       "judge": 6.830025089827052,
       "standard": 0.43969493062997467
      }
     }
    }
   },
   "jevbench_score": 0.7239454481982587,
   "key": "purplemist-mini-1.0",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (model card), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "olaverse/PurpleMIST-Mini-1.0@8072361ec4b632b3b8a73899022281d5de3c639b",
   "noul_decisive": {
    "acc_among_decisive": 0.875,
    "decisive": 32,
    "decisive_rate": 0.08579088471849866,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.68645539289165,
     "cost": 70.75879834655666,
     "intelligence": 8.37974071175967,
     "speed": 58.74735259439709
    },
    "board_release": "v1.6.43",
    "capability": 43.033098052325656,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.22686402261042818,
      1.5438120583885713
     ],
     "B": [
      0.1574817264398275,
      1.158827406474195
     ],
     "C": [
      0.15754446014613066,
      1.0720917072142853
     ]
    },
    "jevbench_score": 0.6877481478120564,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": 159,
    "rank": 202,
    "ranks": {
     "A": 202,
     "B": 202,
     "C": 202,
     "capability": 197
    },
    "scores": {
     "A": 0.6877481478120564,
     "B": 0.4967854880080621,
     "C": 0.4776028804250391
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 7.354124985990348,
     "p50_s_raw": 3.6020624929951737,
     "p95_s_adjusted": 18.144029109398247,
     "p95_s_raw": 8.997014554699124
    },
    "views": {
     "Balanced 33:33:33": 0.5599313395330286,
     "Emphasis on Accuracy 60:20:20": 0.3608793205471312,
     "Emphasis on Cost 20:20:60": 0.7856568103657684,
     "Emphasis on Speed 20:60:20": 0.7610524081768362,
     "Intelligence only": 0.23537033944273675
    }
   },
   "rank": 244,
   "ranked": true,
   "ranks": {
    "A": 244,
    "B": 244,
    "C": 244,
    "capability": 236
   },
   "remeasure": {
    "disclosure": "Re-measure on the same items: this GPU row replaces the earlier loaded-CPU row of the same weights in full (scores and times) and is the second and last use of these weights on draw v1.6-regular-20261012.",
    "draw": "v1.6-regular-20261012",
    "kind": "re-measure on the same items",
    "replaces": {
     "board_release": "v1.6.43",
     "endpoint_kind": "cpu",
     "in_full": "scores and times"
    }
   },
   "remeasured_in": "v1.7.0",
   "repo": "https://huggingface.co/olaverse/PurpleMIST-Mini-1.0",
   "scores": {
    "A": 0.7239454481982587,
    "B": 0.5113943438884895,
    "C": 0.5027398945821242
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3026298979999865,
    "p50_s_raw": 0.07631494899999325,
    "p95_s_adjusted": 0.4624600151999403,
    "p95_s_raw": 0.15623000759997013
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.22727272727272727,
       "cc": -46.57098659210735,
       "cc_by_type": {
        "choice": 8.333333333333334,
        "noul": -100,
        "score": -48.04629310965538
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.2871287128712871,
       "cc": -28.74423110677036,
       "cc_by_type": {
        "choice": 10.92258748674444,
        "noul": -100,
        "score": 2.844719192944467
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.31683168316831684,
       "cc": -20.82159254385255,
       "cc_by_type": {
        "choice": 15.447154471544714,
        "noul": -85.18518518518519,
        "score": 7.27325308208282
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.375,
       "cc": 0.7629902464480551,
       "cc_by_type": {
        "choice": 42.85714285714286,
        "noul": -60,
        "score": 19.431827882201304
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0.7629902464480551
      },
      "jaggedness": {
       "accuracy": 0.26666666666666666,
       "cc": -29.111630061891763,
       "cc_by_type": {
        "choice": 14.82194417709337,
        "noul": -100,
        "score": -2.156834362768656
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.30990415335463256,
       "cc": -15.461313819540068,
       "cc_by_type": {
        "choice": 22.338645769129176,
        "noul": -71.1340206185567,
        "score": 2.411433390807327
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.18518518518518517,
       "cc": -34.84227281771545,
       "cc_by_type": {
        "choice": -11.340206185567009,
        "noul": -100,
        "score": 6.8133877324206615
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.36,
       "cc": -16.747368914339727,
       "cc_by_type": {
        "choice": 31.529181610694497,
        "noul": -83.33333333333334,
        "score": 1.562044979619659
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.23255813953488372,
       "cc": -38.366193779640724,
       "cc_by_type": {
        "choice": 6.666666666666665,
        "noul": -93.33333333333333,
        "score": -28.43191467225552
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.3958333333333333,
       "cc": 22.769640479360852,
       "cc_by_type": {
        "choice": 22.769640479360852
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 22.769640479360852
      },
      "perturb_pair": {
       "accuracy": 0.40625,
       "cc": -15.428764708331562,
       "cc_by_type": {
        "choice": 27.92792792792793,
        "noul": -100,
        "score": 25.785777947077392
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.25,
       "cc": -24.64856439604378,
       "cc_by_type": {
        "choice": 17.89215686274513,
        "noul": -91.66666666666666,
        "score": -0.1711833842098054
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": 1.6816620252144008,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 0,
        "score": 0.2830813137384425
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 1.6816620252144008
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": -31.578947368421055,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.23076923076923078,
       "cc": -23.875193596572927,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -100,
        "score": -9.413599222898505
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.5,
       "cc": -8.777693755084039,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": -100,
        "score": 13.489927584305406
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "el": {
       "accuracy": 0.06666666666666667,
       "cc": -16.612178963752985,
       "cc_by_type": {
        "choice": -34.07821229050279,
        "noul": -100,
        "score": 84.24167539924383
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.2867072111207645,
       "cc": -23.00560999496884,
       "cc_by_type": {
        "choice": 16.74497504301871,
        "noul": -86.33093525179856,
        "score": 0.5691302238733265
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.3870967741935484,
       "cc": -26.26602201719673,
       "cc_by_type": {
        "choice": 21.696574225122355,
        "noul": -75,
        "score": -25.49464027671253
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -27.614920679988757,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": 15.515893697738647
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5,
       "cc": -8.815298731082237,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": -100,
        "score": 15.907044983223884
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.2857142857142857,
       "cc": -32.79394639709711,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -100,
        "score": -31.34273304603994
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.4,
       "cc": -23.36579858731335,
       "cc_by_type": {
        "choice": 43.88152766952456,
        "noul": -100,
        "score": -13.978923431464608
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.4,
       "cc": 0.5067956355677316,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -40,
        "score": -1.2913684546947302
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0.5067956355677316
      },
      "ko": {
       "accuracy": 0.5,
       "cc": -26.28899664672279,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": 10.02189894872052
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.42857142857142855,
       "cc": -13.785099569294184,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": -100,
        "score": -8.021965374549222
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.1,
       "cc": -45.91148210810377,
       "cc_by_type": {
        "choice": -3.8961038961038925,
        "noul": -100,
        "score": -33.838342428207405
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.41025641025641024,
       "cc": -11.560421806782324,
       "cc_by_type": {
        "choice": 54.29538721963606,
        "noul": -80,
        "score": -8.97665263998304
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.32142857142857145,
       "cc": -2.352170896253245,
       "cc_by_type": {
        "choice": 4.537418974661164,
        "noul": -19.999999999999996,
        "score": 8.406068336579098
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 1.4666355934971638,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": -100,
        "score": 29.931821674108505
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 1.4666355934971638
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -34.881619041434675,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": -17.68833538517358
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": -5.785134578904798,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": 47.16072529554367
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.3,
       "cc": -51.780892923305935,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100,
        "score": -79.07149232923985
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.38,
       "cc": 19.918100231539075,
       "cc_by_type": {
        "choice": 19.918100231539075
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 19.918100231539075
      },
      "noul": {
       "accuracy": 0.07466666666666667,
       "cc": -85.06666666666666,
       "cc_by_type": {
        "noul": -85.06666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.37333333333333335,
       "cc": 0.5430206313936092,
       "cc_by_type": {
        "score": 0.5430206313936092
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.5430206313936092
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -24.764293855311823,
       "cc_by_type": {
        "choice": 4,
        "noul": -75,
        "score": -3.2928815659354793
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -25.361393608578854,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -100,
        "score": -20.183559707724143
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.07142857142857142,
       "cc": -49.2889388162784,
       "cc_by_type": {
        "choice": -13.821138211382115,
        "noul": -100,
        "score": -34.0456782374531
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.11764705882352941,
       "cc": -33.27747339938294,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": 5.430737696588029
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.29411764705882354,
       "cc": -29.86268784562891,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": -100,
        "score": -17.12429542094469
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.21052631578947367,
       "cc": -15.19949976003395,
       "cc_by_type": {
        "choice": -13.207547169811319,
        "noul": -100,
        "score": 67.60904788970947
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.36666666666666664,
       "cc": -12.91912658755326,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -100,
        "score": 10.262228080477476
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.18181818181818182,
       "cc": -32.5134196549177,
       "cc_by_type": {
        "choice": 5.9233449477351945,
        "noul": -100,
        "score": -3.463603912488278
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0,
       "cc": -34.95653361358024,
       "cc_by_type": {
        "choice": -25,
        "noul": -100,
        "score": 20.13039915925927
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.09090909090909091,
       "cc": -33.2852225600732,
       "cc_by_type": {
        "choice": -15.631691648822272,
        "noul": -66.66666666666667,
        "score": -17.557309364730656
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.32,
       "cc": -11.739809438644576,
       "cc_by_type": {
        "choice": 53.48837209302325,
        "noul": -77.77777777777779,
        "score": -10.930022631179192
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -38.562807098271385,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100,
        "score": -18.91422774642706
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.3333333333333333,
       "cc": -22.545583669993736,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -100,
        "score": 1.4611376656425978
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.3686635944700461,
       "cc": -8.507105822049263,
       "cc_by_type": {
        "choice": 35.71428571428574,
        "noul": -63.63636363636363,
        "score": 2.4007604559301066
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0
      },
      "moderation_trust_safety": {
       "accuracy": 0.3888888888888889,
       "cc": -24.616273520558888,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100,
        "score": -16.267246665323487
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.3888888888888889,
       "cc": -17.059669885447803,
       "cc_by_type": {
        "choice": 16.434540389972145,
        "noul": -100,
        "score": 32.38644995368445
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.2777777777777778,
       "cc": -20.161455748988658,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -100,
        "score": -1.3710667543551436
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.2608695652173913,
       "cc": -21.852496679566332,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -100,
        "score": 35.89178532361983
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.2916666666666667,
       "cc": -24.550847333370474,
       "cc_by_type": {
        "choice": 14.590747330960852,
        "noul": -100,
        "score": 11.756710668927727
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.82617742073542,
      81.04552351199263
     ],
     "I": [
      5.614657721176678,
      11.3087412814786
     ],
     "capability": [
      39.544712798455194,
      45.225828017189656
     ],
     "composite": null,
     "seed": 62
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.68611335416006,
     "capability": 43.0318464940279,
     "intelligence": 8.377579633895749
    },
    "per_type": {
     "choice": {
      "accuracy": 0.38,
      "calibration": 73.99207874399715,
      "cc": 19.918100231539075,
      "cc_by_type": {
       "choice": 19.918100231539075
      },
      "cc_tiered": 20.851483948725576,
      "cc_tiered_open": 21.510322262497397,
      "cc_tiered_sealed": 20.192645634953756,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 19.918100231539075
     },
     "noul": {
      "accuracy": 0.07466666666666667,
      "calibration": 74.88868920077587,
      "cc": -85.06666666666666,
      "cc_by_type": {
       "noul": -85.06666666666666
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.37333333333333335,
      "calibration": 84.17757211770714,
      "cc": 0.5430206313936092,
      "cc_by_type": {
       "score": 0.5430206313936092
      },
      "cc_tiered": 4.281254952961673,
      "cc_tiered_open": 8.562509905923346,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.5430206313936092
     }
    },
    "raw_on_A": {
     "calibration": 72.87562800357448,
     "capability": 40.88784751944124,
     "intelligence": 8.900067035307991
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.5920947151792082,
    "Emphasis on Accuracy 60:20:20": 0.36844403997888536,
    "Emphasis on Cost 20:20:60": 0.8486338564491605,
    "Emphasis on Speed 20:60:20": 0.851588502276403,
    "Intelligence only": 0.23518828519516521
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Chinh Nguyen (ncchinh)",
   "axes": {
    "calibration": 89.97624968733336,
    "cost": 55.868159288206435,
    "intelligence": 73.40135940607429,
    "speed": 86.35482296351647
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05296499999999975,
      "ece_score": 89.40700000000005,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 89.40700000000005,
      "score_before_time_limit": 89.40700000000005,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.08799999999999997,
      "brier": 0.07739600085333337,
      "ece": 0.046341333333333186,
      "n": 375,
      "score": 90.73173333333337,
      "score_before_time_limit": 90.73173333333337,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9413333333333334,
      "interval90_width": 0.21599999999999997,
      "n": 375,
      "nrps": 0.054102618759999964,
      "score": 89.79001572866669,
      "score_before_time_limit": 89.79001572866669,
      "time_limit_failed_share": 0,
      "top_ece": 0.07504853333333314
     }
    },
    "score": 89.97624968733336
   },
   "capability": 81.68880454670384,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 803,169 input + 0 output tokens over 1,624 decisions = 494.562192118 tokens/decision. GPU-time cost USD 0.0295843040363505/1,000; old list-price cost USD 0.0445105972906404/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0295843040363505/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.029584304036350536
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Quyet-1.0-Large (Chinh Nguyen, Gemma-4-31B decoder, option-letter logits)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (paid evaluation)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 75.23869318243646,
    "I_sealed": 71.56402562971212,
    "base": 73.40135940607429,
    "excess": 1.1012799102861002,
    "gap": 3.674667552724344,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 85.01378955328916,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 84.90091223655237,
       "judge": 80.22412656558997,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 71.3768115942029,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 69.32547839981737,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 57.654545454545456,
       "hard": 72.09425925925925,
       "judge": 68.68847457627118,
       "standard": 70.5788888888889
      }
     },
     "sealed|choice": {
      "cc": 82.59643939217574,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 80.65610859728505,
       "judge": 81.3780260707635,
       "standard": 88.91489099642814
      }
     },
     "sealed|noul": {
      "cc": 66.16688233643043,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 45.63106796116505,
       "judge": 76.3157894736842,
       "standard": 83.09859154929578
      }
     },
     "sealed|score": {
      "cc": 65.9287551605302,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 73.00248226950355,
       "hard": 67.1347549019608,
       "judge": 65.63335329341317,
       "standard": 60.42299492385786
      }
     }
    }
   },
   "jevbench_score": 73.77734958545854,
   "key": "quyet-1-0-large",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9298245614035088,
    "decisive_rate": 0.912,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.97624968733336,
     "cost": 50.546097335562465,
     "intelligence": 73.40135940607429,
     "speed": 86.87440406834797
    },
    "board_release": "v1.6.43",
    "capability": 81.68880454670384,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 4,
    "capability_rank": 8,
    "composite_ci95": {
     "A": [
      69.83728305350246,
      72.20621881721509
     ],
     "B": [
      69.73317863481549,
      73.04885527628076
     ],
     "C": [
      69.83728305350246,
      72.20621881721509
     ]
    },
    "jevbench_score": 71.38706329141472,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 7,
    "rank": 19,
    "ranks": {
     "A": 19,
     "B": 12,
     "C": 19,
     "capability": 18
    },
    "scores": {
     "A": 71.38706329141472,
     "B": 71.78102942459306,
     "C": 71.38706329141472
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3815564611926675,
     "p50_s_raw": 0.11577823059633374,
     "p95_s_adjusted": 0.5382707504555583,
     "p95_s_raw": 0.19413537522777913
    },
    "views": {
     "Balanced 33:33:33": 66.78760078772265,
     "Emphasis on Accuracy 60:20:20": 69.28473442746802,
     "Emphasis on Cost 20:20:60": 59.18115107083417,
     "Emphasis on Speed 20:60:20": 73.59406855476429,
     "Intelligence only": 73.40135940607429
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 35,
   "ranked": true,
   "ranks": {
    "A": 35,
    "B": 20,
    "C": 30,
    "capability": 23
   },
   "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Large",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.77734958545854,
    "B": 73.70184371019474,
    "C": 73.77734958545854
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3948847154155374,
    "p50_s_raw": 0.12244235770776868,
    "p95_s_adjusted": 0.5862021130509675,
    "p95_s_raw": 0.21810105652548373
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-31B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.58390587934255,
      91.370785899262
     ],
     "I": [
      69.18945219901103,
      76.93636317458929
     ],
     "capability": [
      77.9843219931321,
      83.55754626687737
     ],
     "composite": null,
     "seed": 86
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 89.97624968733336,
     "capability": 81.68880454670384,
     "intelligence": 73.40135940607429
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8706666666666667,
      "calibration": 89.40700000000005,
      "cc": 83.46011386565368,
      "cc_by_type": {
       "choice": 83.46011386565368
      },
      "cc_tiered": 83.80511447273244,
      "cc_tiered_open": 85.01378955328916,
      "cc_tiered_sealed": 82.59643939217574,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.46011386565368
     },
     "noul": {
      "accuracy": 0.848,
      "calibration": 90.73173333333337,
      "cc": 69.6,
      "cc_by_type": {
       "noul": 69.6
      },
      "cc_tiered": 68.77184696531667,
      "cc_tiered_open": 71.3768115942029,
      "cc_tiered_sealed": 66.16688233643043,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 69.6
     },
     "score": {
      "accuracy": 0.7493333333333333,
      "calibration": 89.79001572866669,
      "cc": 66.83812280701756,
      "cc_by_type": {
       "score": 66.83812280701756
      },
      "cc_tiered": 67.6271167801738,
      "cc_tiered_open": 69.32547839981737,
      "cc_tiered_sealed": 65.9287551605302,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 66.83812280701756
     }
    },
    "raw_on_A": {
     "calibration": 86.55147389375357,
     "capability": 81.17122174668583,
     "intelligence": 75.7909695996181
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.6005008285897,
    "Emphasis on Accuracy 60:20:20": 71.07260946246477,
    "Emphasis on Cost 20:20:60": 63.369987278116994,
    "Emphasis on Speed 20:60:20": 75.45644013475498,
    "Intelligence only": 73.40135940607429
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Chinh Nguyen (ncchinh)",
   "axes": {
    "calibration": 76.76007761870373,
    "cost": 77.97650072588999,
    "intelligence": 41.73350604436517,
    "speed": 89.14092560273156
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.15434999999999968,
      "ece_score": 69.13000000000007,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 69.13000000000007,
      "score_before_time_limit": 69.13000000000007,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.24,
      "brier": 0.16916629888000007,
      "ece": 0.10974879999999991,
      "n": 375,
      "score": 78.05024000000002,
      "score_before_time_limit": 78.05024000000002,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.92,
      "interval90_width": 0.4133333333333333,
      "n": 375,
      "nrps": 0.10737614287777777,
      "score": 83.09999285611111,
      "score_before_time_limit": 83.09999285611111,
      "time_limit_failed_share": 0,
      "top_ece": 0.11531199999999998
     }
    },
    "score": 76.76007761870373
   },
   "capability": 59.24679183153445,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 896,009 input + 0 output tokens over 1,624 decisions = 551.729679803 tokens/decision. GPU-time cost USD 0.00542146480124741/1,000; old list-price cost USD 0.0165518903940887/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00542146480124741/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005421464801247415
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Quyet-1.0-Medium (Chinh Nguyen, Qwen3.5-4B decoder, option-letter logits)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (paid evaluation)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 42.75300136168062,
    "I_sealed": 40.71401072704972,
    "base": 41.73350604436517,
    "excess": -0.5343970078073448,
    "gap": 2.038990634630899,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.4971468014646,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 67.9144385026738,
       "judge": 72.31377719182596,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 16.34230503795721,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 4.347826086956519,
       "judge": 0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 41.41955224562005,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.18969696969697,
       "hard": 35.19814814814814,
       "judge": 54.641694915254234,
       "standard": 31.14407407407409
      }
     },
     "sealed|choice": {
      "cc": 59.00724128213771,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 55.20361990950226,
       "judge": 55.30726256983241,
       "standard": 68.96169478999877
      }
     },
     "sealed|noul": {
      "cc": 25.985534052552406,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 60.00000000000001,
       "hard": 12.621359223300965,
       "judge": 26.315789473684205,
       "standard": 35.2112676056338
      }
     },
     "sealed|score": {
      "cc": 37.14925684645904,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 55.4977304964539,
       "hard": 32.33151960784312,
       "judge": 40.893173652694614,
       "standard": 31.994619289340076
      }
     }
    }
   },
   "jevbench_score": 45.659799874048836,
   "key": "quyet-1-0-medium",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8421052631578947,
    "decisive_rate": 0.76,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 76.76007761870373,
     "cost": 63.43457194636937,
     "intelligence": 41.73350604436517,
     "speed": 89.46638461924755
    },
    "board_release": "v1.6.43",
    "capability": 59.24679183153445,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 62,
    "capability_rank": 87,
    "composite_ci95": {
     "A": [
      30.84848869900701,
      57.23347326869325
     ],
     "B": [
      27.312995844624446,
      52.90407172237297
     ],
     "C": [
      21.422561596532642,
      39.74546754770365
     ]
    },
    "jevbench_score": 43.58925238992216,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 48,
    "rank": 73,
    "ranks": {
     "A": 73,
     "B": 77,
     "C": 77,
     "capability": 135
    },
    "scores": {
     "A": 43.58925238992216,
     "B": 39.63222739545648,
     "C": 30.270314159668164
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3048233319917927,
     "p50_s_raw": 0.07741166599589633,
     "p95_s_adjusted": 0.3709482546968502,
     "p95_s_raw": 0.1104741273484251
    },
    "views": {
     "Balanced 33:33:33": 41.05875168344406,
     "Emphasis on Accuracy 60:20:20": 35.24739388678317,
     "Emphasis on Cost 20:20:60": 42.25762883934701,
     "Emphasis on Speed 20:60:20": 47.54937088611952,
     "Intelligence only": 29.074657383286326
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 128,
   "ranked": true,
   "ranks": {
    "A": 128,
    "B": 135,
    "C": 133,
    "capability": 169
   },
   "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Medium",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 45.659799874048836,
    "B": 40.98406740502822,
    "C": 31.708194356978357
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3133888949872926,
    "p50_s_raw": 0.0816944474936463,
    "p95_s_adjusted": 0.38888736818771574,
    "p95_s_raw": 0.11944368409385787
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.2671834460676,
      80.2774031670074
     ],
     "I": [
      36.06269585061453,
      46.7313636734712
     ],
     "capability": [
      54.568687859056396,
      62.84407796698955
     ],
     "composite": null,
     "seed": 87
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 76.76007761870373,
     "capability": 59.24679183153445,
     "intelligence": 41.73350604436517
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7013333333333334,
      "calibration": 69.13000000000007,
      "cc": 61.804799029963135,
      "cc_by_type": {
       "choice": 61.804799029963135
      },
      "cc_tiered": 64.75219404180115,
      "cc_tiered_open": 70.4971468014646,
      "cc_tiered_sealed": 59.00724128213771,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 61.804799029963135
     },
     "noul": {
      "accuracy": 0.64,
      "calibration": 78.05024000000002,
      "cc": 28.000000000000004,
      "cc_by_type": {
       "noul": 28.000000000000004
      },
      "cc_tiered": 21.163919545254807,
      "cc_tiered_open": 16.34230503795721,
      "cc_tiered_sealed": 25.985534052552406,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 28.000000000000004
     },
     "score": {
      "accuracy": 0.56,
      "calibration": 83.09999285611111,
      "cc": 38.22100877192973,
      "cc_by_type": {
       "score": 38.22100877192973
      },
      "cc_tiered": 39.28440454603954,
      "cc_tiered_open": 41.41955224562005,
      "cc_tiered_sealed": 37.14925684645904,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 38.22100877192973
     }
    },
    "raw_on_A": {
     "calibration": 77.01676225205841,
     "capability": 61.22371994730187,
     "intelligence": 45.430677642545334
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 43.538391666048604,
    "Emphasis on Accuracy 60:20:20": 36.312631497478954,
    "Emphasis on Cost 20:20:60": 47.29442888919889,
    "Emphasis on Speed 20:60:20": 49.451244107908806,
    "Intelligence only": 29.074657383286326
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Chinh Nguyen (ncchinh)",
   "axes": {
    "calibration": 60.96568104722222,
    "cost": 79.56261245432609,
    "intelligence": 4.019045994532796,
    "speed": 94.90280801143004
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.22390249999999998,
      "ece_score": 55.2195,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 55.2195,
      "score_before_time_limit": 55.2195,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.712,
      "brier": 0.28332537661333335,
      "ece": 0.2195469333333333,
      "n": 375,
      "score": 56.09061333333334,
      "score_before_time_limit": 56.09061333333334,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9253333333333333,
      "interval90_width": 0.8177777777777777,
      "n": 375,
      "nrps": 0.1894891371666667,
      "score": 71.58692980833334,
      "score_before_time_limit": 71.58692980833334,
      "time_limit_failed_share": 0,
      "top_ece": 0.18938613333333326
     }
    },
    "score": 60.96568104722222
   },
   "capability": 32.492363520877504,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.020959591958551866,
     0.3032092824492544
    ],
    "B": [
     0.01370001323335493,
     0.2103075700089039
    ],
    "C": [
     0.014555272193438792,
     0.21056200170087108
    ]
   },
   "cost": {
    "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra same-size encoders (bge-large, e5-large, Qwen3-Embedding-0.6B) $0.01/M input (SEA-LION-ModernBERT-300M has no market listing; the ModernBERT-large-class reference of the Laya/Certo/OpenDecision rows) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004800055418719211
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Quyet-1.0-Small (Chinh Nguyen, SEA-LION-ModernBERT-300M encoder with fixed heads)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (paid evaluation)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 4.071843176603614,
    "I_sealed": 3.966248812461978,
    "base": 4.019045994532796,
    "excess": -2.467793278296608,
    "gap": 0.10559436414163592,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 12.215529529810842,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 2.702702702702708,
       "hard": 5.630701478452354,
       "judge": 20.896506262359924,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -71.42857142857143,
       "hard": -56.52173913043479,
       "judge": -40,
       "standard": -55.55555555555556
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 13.416060606060597,
       "hard": -26.448333333333316,
       "judge": -3.2088135593220546,
       "standard": -76.01518518518517
      }
     },
     "sealed|choice": {
      "cc": 11.898746437385935,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 17.522725593082576,
       "hard": 5.316742081447947,
       "judge": 19.92551210428307,
       "standard": 10.210617071067881
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -72,
       "hard": -59.22330097087378,
       "judge": -65.78947368421053,
       "standard": -46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 20.691063829787236,
       "hard": -7.6461764705882285,
       "judge": -0.1282634730539245,
       "standard": 0.04786802030454229
      }
     }
    }
   },
   "jevbench_score": 0.08963662915567427,
   "key": "quyet-1-0-small",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7129629629629629,
    "decisive_rate": 0.288,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 60.96568104722222,
     "cost": 79.56261245432609,
     "intelligence": 4.019045994532796,
     "speed": 94.91846454857637
    },
    "board_release": "v1.6.43",
    "capability": 32.492363520877504,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 127,
    "capability_rank": 156,
    "composite_ci95": {
     "A": [
      0.020959672741331675,
      0.3032118958147926
     ],
     "B": [
      0.01370004084448916,
      0.2103085758125444
     ],
     "C": [
      0.014555328292591438,
      0.21056381653805042
     ]
    },
    "jevbench_score": 0.08963716950381444,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 178,
    "rank": 222,
    "ranks": {
     "A": 222,
     "B": 222,
     "C": 222,
     "capability": 224
    },
    "scores": {
     "A": 0.08963716950381444,
     "B": 0.0601437708059232,
     "C": 0.06224803437764891
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.17903870801674202,
     "p50_s_raw": 0.01451935400837101,
     "p95_s_adjusted": 0.17997269080078693,
     "p95_s_raw": 0.014986345400393473
    },
    "views": {
     "Balanced 33:33:33": 0.07128318534334778,
     "Emphasis on Accuracy 60:20:20": 0.04197968509922768,
     "Emphasis on Cost 20:20:60": 0.10875180747478416,
     "Emphasis on Speed 20:60:20": 0.11026128891416027,
     "Intelligence only": 0.025967427058159954
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 264,
   "ranked": true,
   "ranks": {
    "A": 264,
    "B": 264,
    "C": 264,
    "capability": 265
   },
   "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Small",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.08963662915567427,
    "B": 0.06014357619379634,
    "C": 0.06224765913588491
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.17911875999416224,
    "p50_s_raw": 0.014559379997081123,
    "p95_s_adjusted": 0.18054194830911,
    "p95_s_raw": 0.015270974154555006
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "aisingapore/SEA-LION-ModernBERT-300M",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      55.688303752892594,
      64.99464873956296
     ],
     "I": [
      2.4305093708220933,
      6.1818340512057315
     ],
     "capability": [
      29.486356353832495,
      35.15105810056002
     ],
     "composite": {
      "A": [
       0.020959591958551866,
       0.3032092824492544
      ],
      "B": [
       0.01370001323335493,
       0.2103075700089039
      ],
      "C": [
       0.014555272193438792,
       0.21056200170087108
      ]
     },
     "seed": 88
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 60.96568104722222,
     "capability": 32.492363520877504,
     "intelligence": 4.019045994532796
    },
    "per_type": {
     "choice": {
      "accuracy": 0.30533333333333335,
      "calibration": 55.2195,
      "cc": 11.162054886655328,
      "cc_by_type": {
       "choice": 11.162054886655328
      },
      "cc_tiered": 12.057137983598388,
      "cc_tiered_open": 12.215529529810842,
      "cc_tiered_sealed": 11.898746437385935,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 11.162054886655328
     },
     "noul": {
      "accuracy": 0.20533333333333334,
      "calibration": 56.09061333333334,
      "cc": -58.93333333333332,
      "cc_by_type": {
       "noul": -58.93333333333332
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2986666666666667,
      "calibration": 71.58692980833334,
      "cc": -4.271587719298253,
      "cc_by_type": {
       "score": -4.271587719298253
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 55.62018953114671,
     "capability": 29.685347509548446,
     "intelligence": 3.750505487950187
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.07128272971559697,
    "Emphasis on Accuracy 60:20:20": 0.041979590286340934,
    "Emphasis on Cost 20:20:60": 0.10875117117698081,
    "Emphasis on Speed 20:60:20": 0.11025932668532612,
    "Intelligence only": 0.025967427058159954
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Chinh Nguyen (ncchinh)",
   "axes": {
    "calibration": 73.02134054125926,
    "cost": 89.11296503596958,
    "intelligence": 6.054813340422714,
    "speed": 94.88950688832834
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1464534374999999,
      "ece_score": 70.70931250000002,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 70.70931250000002,
      "score_before_time_limit": 70.70931250000002,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.72,
      "brier": 0.2701432261600001,
      "ece": 0.15907813333333334,
      "n": 375,
      "score": 68.18437333333334,
      "score_before_time_limit": 68.18437333333334,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9653333333333334,
      "interval90_width": 0.8146666666666667,
      "n": 375,
      "nrps": 0.1639986175244445,
      "score": 80.17033579044444,
      "score_before_time_limit": 80.17033579044444,
      "time_limit_failed_share": 0,
      "top_ece": 0.11629733333333336
     }
    },
    "score": 73.02134054125926
   },
   "capability": 39.53807694084099,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.05507555671906829,
     0.8570652295031572
    ],
    "B": [
     0.036403719413010745,
     0.6085722574185467
    ],
    "C": [
     0.038246914388241855,
     0.5951841871549702
    ]
   },
   "cost": {
    "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra base-size encoders (bge-base, e5-base, gte-base, all-mpnet-base) $0.005/M input (ModernBERT-base has no market listing; base-size encoder reference) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0023062222906403943
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Quyet-1.0-Small-EN (Chinh Nguyen, ModernBERT-base encoder with fixed heads)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (paid evaluation)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 5.966953908358329,
    "I_sealed": 6.142672772487099,
    "base": 6.054813340422714,
    "excess": -2.749106506567014,
    "gap": -0.17571886412877014,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 11.596876448581234,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486491,
       "hard": 24.50456118276188,
       "judge": 1.1206328279499007,
       "standard": 2.051067392214312
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -71.42857142857143,
       "hard": -65.21739130434783,
       "judge": -70,
       "standard": -44.44444444444444
      }
     },
     "open|score": {
      "cc": 6.303985276493753,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 12.853939393939395,
       "hard": 15.763981481481492,
       "judge": 4.772711864406753,
       "standard": -13.594074074074047
      }
     },
     "sealed|choice": {
      "cc": 14.421677328746334,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 4.219939398418472,
       "hard": 11.934389140271476,
       "judge": 14.338919925512121,
       "standard": 24.621258775711308
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -48,
       "hard": -84.46601941747574,
       "judge": -52.63157894736843,
       "standard": -60.56338028169015
      }
     },
     "sealed|score": {
      "cc": 4.006340988714965,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 17.547730496453894,
       "hard": 1.5428921568626852,
       "judge": 5.2876646706586765,
       "standard": 0.240558375634492
      }
     }
    }
   },
   "jevbench_score": 0.2923902780429571,
   "key": "quyet-1-0-small-en",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.638095238095238,
    "decisive_rate": 0.28,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.02134054125926,
     "cost": 89.11296503596958,
     "intelligence": 6.054813340422714,
     "speed": 94.90893356488051
    },
    "board_release": "v1.6.43",
    "capability": 39.53807694084099,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 114,
    "capability_rank": 142,
    "composite_ci95": {
     "A": [
      0.0550759146094291,
      0.8570777384335173
     ],
     "B": [
      0.03640384449772412,
      0.6085773029152636
     ],
     "C": [
      0.03824716292321466,
      0.5951928739121649
     ]
    },
    "jevbench_score": 0.29239342204088814,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 167,
    "rank": 211,
    "ranks": {
     "A": 211,
     "B": 211,
     "C": 211,
     "capability": 206
    },
    "scores": {
     "A": 0.29239342204088814,
     "B": 0.20045885649425463,
     "C": 0.20305098752839457
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1791461439919658,
     "p50_s_raw": 0.0145730719959829,
     "p95_s_adjusted": 0.18025992200709878,
     "p95_s_raw": 0.015129961003549396
    },
    "views": {
     "Balanced 33:33:33": 0.23536198594902974,
     "Emphasis on Accuracy 60:20:20": 0.14175760897818884,
     "Emphasis on Cost 20:20:60": 0.3502184575801078,
     "Emphasis on Speed 20:60:20": 0.3525263048935194,
     "Intelligence only": 0.08878963459745969
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 252,
   "ranked": true,
   "ranks": {
    "A": 252,
    "B": 252,
    "C": 252,
    "capability": 248
   },
   "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Small-EN",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.2923902780429571,
    "B": 0.20045767429781194,
    "C": 0.203048804196498
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.17937943699362222,
    "p50_s_raw": 0.01468971849681111,
    "p95_s_adjusted": 0.18083257000980665,
    "p95_s_raw": 0.01541628500490333
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "answerdotai/ModernBERT-base",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.50774946138796,
      76.70202320961018
     ],
     "I": [
      3.379327510519746,
      8.897820124597144
     ],
     "capability": [
      36.01781590414684,
      42.01829735190088
     ],
     "composite": {
      "A": [
       0.05507555671906829,
       0.8570652295031572
      ],
      "B": [
       0.036403719413010745,
       0.6085722574185467
      ],
      "C": [
       0.038246914388241855,
       0.5951841871549702
      ]
     },
     "seed": 89
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.02134054125926,
     "capability": 39.53807694084099,
     "intelligence": 6.054813340422714
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3253333333333333,
      "calibration": 70.70931250000002,
      "cc": 13.719769237327435,
      "cc_by_type": {
       "choice": 13.719769237327435
      },
      "cc_tiered": 13.009276888663784,
      "cc_tiered_open": 11.596876448581234,
      "cc_tiered_sealed": 14.421677328746334,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.719769237327435
     },
     "noul": {
      "accuracy": 0.17866666666666667,
      "calibration": 68.18437333333334,
      "cc": -64.26666666666667,
      "cc_by_type": {
       "noul": -64.26666666666667
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.312,
      "calibration": 80.17033579044444,
      "cc": 5.329298245614,
      "cc_by_type": {
       "score": 5.329298245614
      },
      "cc_tiered": 5.155163132604359,
      "cc_tiered_open": 6.303985276493753,
      "cc_tiered_sealed": 4.006340988714965,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 5.329298245614
     }
    },
    "raw_on_A": {
     "calibration": 75.9241654666809,
     "capability": 42.17966881905028,
     "intelligence": 8.435172171419651
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.23535926977189042,
    "Emphasis on Accuracy 60:20:20": 0.14175701778059244,
    "Emphasis on Cost 20:20:60": 0.35021484917491197,
    "Emphasis on Speed 20:60:20": 0.35251533676546687,
    "Intelligence only": 0.08878963459745969
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Chinh Nguyen (ncchinh)",
   "axes": {
    "calibration": 58.46458026762962,
    "cost": 88.43590631389968,
    "intelligence": 3.6964771541368573,
    "speed": 95.07796177344144
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.2315684375,
      "ece_score": 53.6863125,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 53.6863125,
      "score_before_time_limit": 53.6863125,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.8026666666666666,
      "brier": 0.28768576031999976,
      "ece": 0.19567466666666677,
      "n": 375,
      "score": 60.86506666666664,
      "score_before_time_limit": 60.86506666666664,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8933333333333333,
      "interval90_width": 0.7453333333333331,
      "n": 375,
      "nrps": 0.1976125006088888,
      "score": 60.84236163622223,
      "score_before_time_limit": 60.84236163622223,
      "time_limit_failed_share": 0,
      "top_ece": 0.2927701333333333
     }
    },
    "score": 58.46458026762962
   },
   "capability": 31.08052871088324,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.019766201285679498,
     0.27340010410511306
    ],
    "B": [
     0.012911538304013407,
     0.1889551228714117
    ],
    "C": [
     0.013726528670610767,
     0.18986118340632852
    ]
   },
   "cost": {
    "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; self-hosted open weights, no public tariff): DeepInfra base-size encoders (bge-base, e5-base, gte-base, all-mpnet-base) $0.005/M input (mmBERT-small 16-layer, 183M; base-size class reference, errs high vs the small-encoder $0.004 class) x the package's own measured usage.input_tokens; one forward pass, nothing generated; run in process on 1x H100 80GB (Lium), quyet 1.0.0; estimated, not charged",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0024292364532019703
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Quyet-1.0-Tiny (Chinh Nguyen, mmBERT-small encoder (16 layers kept) with fixed heads)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (paid evaluation)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 1.2549826648038989,
    "I_sealed": 6.1379716434698155,
    "base": 3.6964771541368573,
    "excess": -7.456376621104161,
    "gap": -4.882988978665916,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 3.764947994411697,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -12.86486486486486,
       "hard": 5.630701478452354,
       "judge": 12.986156888595913,
       "standard": -5.4834658853076625
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -71.42857142857143,
       "hard": -56.52173913043479,
       "judge": -80,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -8.307575757575748,
       "hard": -21.650555555555574,
       "judge": -15.678474576271228,
       "standard": -51.14888888888891
      }
     },
     "sealed|choice": {
      "cc": 16.468453763405932,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 12.201611115216933,
       "hard": 14.479638009049756,
       "judge": 18.06331471135942,
       "standard": 20.18721517428256
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -84,
       "hard": -68.93203883495146,
       "judge": -71.05263157894737,
       "standard": -83.09859154929578
      }
     },
     "sealed|score": {
      "cc": 1.9454611670035158,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 16.69851063829787,
       "hard": -2.889509803921597,
       "judge": -1.9068263473054348,
       "standard": 10.017309644669991
      }
     }
    }
   },
   "jevbench_score": 0.07064720497480408,
   "key": "quyet-1-0-tiny",
   "last_measured_on": "2026-10-04",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6351351351351351,
    "decisive_rate": 0.19733333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 58.46458026762962,
     "cost": 88.43590631389968,
     "intelligence": 3.6964771541368573,
     "speed": 95.09610910731021
    },
    "board_release": "v1.6.43",
    "capability": 31.08052871088324,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 128,
    "capability_rank": 157,
    "composite_ci95": {
     "A": [
      0.019766287637893492,
      0.2734027471253872
     ],
     "B": [
      0.012911567780248491,
      0.18895613284128507
     ],
     "C": [
      0.013726588637426037,
      0.1898630188370744
     ]
    },
    "jevbench_score": 0.07064766318734213,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 180,
    "rank": 224,
    "ranks": {
     "A": 224,
     "B": 224,
     "C": 224,
     "capability": 226
    },
    "scores": {
     "A": 0.07064766318734213,
     "B": 0.04711839739648976,
     "C": 0.04906087721343202
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1753336349996971,
     "p50_s_raw": 0.012666817499848548,
     "p95_s_adjusted": 0.17641026510682423,
     "p95_s_raw": 0.013205132553412113
    },
    "views": {
     "Balanced 33:33:33": 0.05608574548219273,
     "Emphasis on Accuracy 60:20:20": 0.03279057174059279,
     "Emphasis on Cost 20:20:60": 0.08676445199059092,
     "Emphasis on Speed 20:60:20": 0.08720297577636826,
     "Intelligence only": 0.020203381773039073
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 266,
   "ranked": true,
   "ranks": {
    "A": 266,
    "B": 266,
    "C": 266,
    "capability": 266
   },
   "repo": "https://huggingface.co/chinhnc/Quyet-1.0-Tiny",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.07064720497480408,
    "B": 0.0471182343378976,
    "C": 0.0490605590102806
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.1755638730013743,
    "p50_s_raw": 0.012781936500687152,
    "p95_s_adjusted": 0.17691663471632635,
    "p95_s_raw": 0.013458317358163184
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "jhu-clsp/mmBERT-small",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      52.6105536767426,
      62.31255872829814
     ],
     "I": [
      2.382388771227772,
      5.956289831309647
     ],
     "capability": [
      27.937688273976466,
      33.75728623357733
     ],
     "composite": {
      "A": [
       0.019766201285679498,
       0.27340010410511306
      ],
      "B": [
       0.012911538304013407,
       0.1889551228714117
      ],
      "C": [
       0.013726528670610767,
       0.18986118340632852
      ]
     },
     "seed": 90
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 58.46458026762962,
     "capability": 31.08052871088324,
     "intelligence": 3.6964771541368573
    },
    "per_type": {
     "choice": {
      "accuracy": 0.32133333333333336,
      "calibration": 53.6863125,
      "cc": 13.20822636719302,
      "cc_by_type": {
       "choice": 13.20822636719302
      },
      "cc_tiered": 10.116700878908814,
      "cc_tiered_open": 3.764947994411697,
      "cc_tiered_sealed": 16.468453763405932,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.20822636719302
     },
     "noul": {
      "accuracy": 0.12533333333333332,
      "calibration": 60.86506666666664,
      "cc": -74.93333333333334,
      "cc_by_type": {
       "noul": -74.93333333333334
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.33866666666666667,
      "calibration": 60.84236163622223,
      "cc": -1.831508771929835,
      "cc_by_type": {
       "score": -1.831508771929835
      },
      "cc_tiered": 0.9727305835017579,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 1.9454611670035158,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 54.460514049074064,
     "capability": 29.223962167243283,
     "intelligence": 3.987410285412503
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.05608536043413992,
    "Emphasis on Accuracy 60:20:20": 0.03279049277078114,
    "Emphasis on Cost 20:20:60": 0.08676389909254054,
    "Emphasis on Speed 20:60:20": 0.08720130029466011,
    "Intelligence only": 0.020203381773039073
   }
  },
  {
   "adapter_id": "images/qwen3-reranker-4b/spec.json:reranker_adapter.RerankerAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Qwen",
   "axes": {
    "calibration": 73.15350658389404,
    "cost": 60.56419192241651,
    "intelligence": 16.337944014530347,
    "speed": 81.05912758410997
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12818639012273808,
      "ece_score": 74.36272197545239,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 74.36272197545239,
      "score_before_time_limit": 74.36272197545239,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.7466666666666666,
      "brier": 0.23186747724503004,
      "ece": 0.1932172851672485,
      "n": 375,
      "score": 61.3565429665503,
      "score_before_time_limit": 61.3565429665503,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9093333333333333,
      "interval90_width": 0.6708888888888882,
      "n": 375,
      "nrps": 0.15088039394905625,
      "score": 83.74125480967948,
      "score_before_time_limit": 83.74125480967948,
      "time_limit_failed_share": 0,
      "top_ece": 0.08714725492867716
     }
    },
    "score": 73.15350658389404
   },
   "capability": 44.7457252992122,
   "class": "reranker",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 3,409,751 input + 0 output tokens over 1,624 decisions = 2099.60036946 tokens/decision. GPU-time cost USD 0.020631316234009/1,000; old list-price cost USD 0.0524900092364532/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.020631316234009/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020631316234009007
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Qwen3-Reranker-4B",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 17.351796606180663,
    "I_sealed": 15.324091422880029,
    "base": 16.337944014530347,
    "excess": -0.5456824591376095,
    "gap": 2.0277051833006343,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 36.57581104234733,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.2972972972973,
       "hard": 22.617175212330924,
       "judge": 52.537903757415954,
       "standard": 32.18920050230222
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -91.30434782608697,
       "judge": -40,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 15.479578776194659,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 16.665491764012373,
       "hard": -1.4208863867574584,
       "judge": 33.97314797019466,
       "standard": 20.94719881719004
      }
     },
     "sealed|choice": {
      "cc": 33.418936417285686,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 45.458576601877176,
       "hard": 22.115384615384603,
       "judge": 34.823091247672274,
       "standard": 47.89998768321223
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -43.99999999999999,
       "hard": -84.46601941747574,
       "judge": -57.89473684210527,
       "standard": -43.66197183098591
      }
     },
     "sealed|score": {
      "cc": 12.553337851354403,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 16.14176889876734,
       "hard": 9.253841484765735,
       "judge": 15.752141635882,
       "standard": 12.559909384033885
      }
     }
    }
   },
   "jevbench_score": 4.117471541314488,
   "key": "qwen3-reranker-4b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7473684210526316,
    "decisive_rate": 0.25333333333333335,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.15350658389404,
     "cost": 48.39770052432463,
     "intelligence": 16.337944014530347,
     "speed": 82.14971164581814
    },
    "board_release": "v1.6.43",
    "capability": 44.7457252992122,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 100,
    "capability_rank": 128,
    "composite_ci95": {
     "A": [
      2.2277188584602894,
      5.423744955848025
     ],
     "B": [
      1.7231876454208677,
      4.422291990633802
     ],
     "C": [
      1.5470269850418672,
      3.766489552672239
     ]
    },
    "jevbench_score": 3.715006798009075,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 147,
    "rank": 187,
    "ranks": {
     "A": 187,
     "B": 190,
     "C": 188,
     "capability": 186
    },
    "scores": {
     "A": 3.715006798009075,
     "B": 2.961112195421367,
     "C": 2.5798658319507455
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5581826242618263,
     "p50_s_raw": 0.20409131213091314,
     "p95_s_adjusted": 1.092075143335387,
     "p95_s_raw": 0.47103757166769356
    },
    "views": {
     "Balanced 33:33:33": 3.1912622823260945,
     "Emphasis on Accuracy 60:20:20": 2.310804869814862,
     "Emphasis on Cost 20:20:60": 3.695072544212985,
     "Emphasis on Speed 20:60:20": 4.224997058133206,
     "Intelligence only": 1.6344124017617998
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 229,
   "ranked": true,
   "ranks": {
    "A": 229,
    "B": 229,
    "C": 229,
    "capability": 227
   },
   "repo": "https://huggingface.co/Qwen/Qwen3-Reranker-4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 4.117471541314488,
    "B": 3.236822275269605,
    "C": 2.859355237023949
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5727677877061069,
    "p50_s_raw": 0.21138389385305345,
    "p95_s_adjusted": 1.3680710606742645,
    "p95_s_raw": 0.6090355303371323
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "4B instruction-aware generative reranker",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.92034254293053,
      76.98495141643136
     ],
     "I": [
      13.411885193349915,
      18.933037790738602
     ],
     "capability": [
      41.28670365010265,
      47.260558123220996
     ],
     "composite": null,
     "seed": 91
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.15350658389404,
     "capability": 44.7457252992122,
     "intelligence": 16.337944014530347
    },
    "per_type": {
     "choice": {
      "accuracy": 0.4866666666666667,
      "calibration": 74.36272197545239,
      "cc": 34.35199833274914,
      "cc_by_type": {
       "choice": 34.35199833274914
      },
      "cc_tiered": 34.99737372981651,
      "cc_tiered_open": 36.57581104234733,
      "cc_tiered_sealed": 33.418936417285686,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 34.35199833274914
     },
     "noul": {
      "accuracy": 0.18933333333333333,
      "calibration": 61.3565429665503,
      "cc": -62.133333333333326,
      "cc_by_type": {
       "noul": -62.133333333333326
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.37333333333333335,
      "calibration": 83.74125480967948,
      "cc": 12.388563279917397,
      "cc_by_type": {
       "score": 12.388563279917397
      },
      "cc_tiered": 14.016458313774532,
      "cc_tiered_open": 15.479578776194659,
      "cc_tiered_sealed": 12.553337851354403,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 12.388563279917397
     }
    },
    "raw_on_A": {
     "calibration": 68.75797234609402,
     "capability": 42.734177704587836,
     "intelligence": 16.71038306308165
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 3.556860349095305,
    "Emphasis on Accuracy 60:20:20": 2.5126255169634013,
    "Emphasis on Cost 20:20:60": 4.337545072477729,
    "Emphasis on Speed 20:60:20": 4.65321329197435,
    "Intelligence only": 1.7444245975921615
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 592,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Qwen / Chutes",
   "axes": {
    "calibration": 63.81,
    "cost": 0,
    "intelligence": 33.36,
    "speed": 54.99
   },
   "calibration": {
    "score": 63.81
   },
   "capability": 48.59,
   "class": "llm-baseline",
   "composite_ci95": {
    "A": [
     0,
     0
    ],
    "B": [
     0,
     0
    ],
    "C": [
     0,
     0
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)",
    "kind": "estimate",
    "usd_per_1000": 2.1783887438423646
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 233,
    "n_over_limit_answered_ok": 225,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Qwen3.8 27B (Chutes TEE)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 32.85,
    "I_sealed": 28.72,
    "base": 33.36,
    "per_type_split": {
     "open|choice": {
      "cc": 50.7411,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.3243,
       "hard": 24.5046,
       "judge": 60.4483,
       "standard": 69.8619
      }
     },
     "open|noul": {
      "cc": 47.8226,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 21.7391,
       "judge": 50,
       "standard": 77.7778
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": -46,
       "hard": -52.8611,
       "judge": -44.6102,
       "standard": -18
      }
     },
     "sealed|choice": {
      "cc": 46.11,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 55.5144,
       "hard": 30.195,
       "judge": 59.2965,
       "standard": 53.4583
      }
     },
     "sealed|noul": {
      "cc": 40.0561,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 27.2727,
       "judge": 30,
       "standard": 62.5
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 24.5833,
       "hard": -130.1205,
       "judge": -150,
       "standard": -3.5
      }
     }
    }
   },
   "jevbench_score": 0,
   "key": "qwen3.8-27b",
   "last_measured_on": "2026-10-06",
   "licence": "open weights",
   "listing": "ranked",
   "method": "v1.7",
   "open": "weights",
   "previous": {
    "api_board_rank": 45,
    "axes": {
     "calibration": 100,
     "cost": 0,
     "intelligence": 96.38,
     "speed": 57.43
    },
    "board_release": "v1.6.43",
    "capability": 98.19,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0,
      0
     ],
     "B": [
      0,
      0
     ],
     "C": [
      0,
      0
     ]
    },
    "jevbench_score": 0,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 242,
    "ranks": {
     "A": 242,
     "B": 242,
     "C": 242,
     "capability": 2
    },
    "scores": {
     "A": 0,
     "B": 0,
     "C": 0
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 592,
     "p50_s_adjusted": 7.095,
     "p50_s_raw": 7.095,
     "p95_s_adjusted": 25.469,
     "p95_s_raw": 25.469
    },
    "views": null
   },
   "rank": 283,
   "ranked": true,
   "ranks": {
    "A": 283,
    "B": 283,
    "C": 283,
    "capability": 216
   },
   "repo": "https://benchmarkheaven.com/jev-models/qwen3.8-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0,
    "B": 0,
    "C": 0
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 8.35,
    "p50_s_raw": 8.35,
    "p95_s_adjusted": 37.933,
    "p95_s_raw": 37.933
   },
   "status": {
    "answered_ok": 592,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "e513ccfd7f429c1bfc4533c40ba34037b9951812bbddb6d01db7e0ef9753386f"
   }
  },
  {
   "adapter_id": "images/qwen35-9b-jev-data-mix-v2/spec.json:qwen35_nimble_lora.Qwen35NimbleLoraAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "jsaurabh",
   "axes": {
    "calibration": 79.78895435560509,
    "cost": 65.35303366044648,
    "intelligence": 41.94184606895763,
    "speed": 85.22302455389745
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14238786731364284,
      "ece_score": 71.52242653727143,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 71.52242653727143,
      "score_before_time_limit": 71.52242653727143,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2911051212938005,
      "brier": 0.12127289774594222,
      "ece": 0.07874545275117417,
      "n": 371,
      "score": 84.25090944976516,
      "score_before_time_limit": 84.25090944976516,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8888888888888888,
      "interval90_width": 0.31820234869015335,
      "n": 369,
      "nrps": 0.0937862889000391,
      "score": 83.59352707977865,
      "score_before_time_limit": 83.59352707977865,
      "time_limit_failed_share": 0,
      "top_ece": 0.11717158475219394
     }
    },
    "score": 79.78895435560509
   },
   "capability": 60.86540021228136,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,049,330 input + 0 output tokens over 1,624 decisions = 646.139162562 tokens/decision. GPU-time cost USD 0.0142856129065212/1,000; old list-price cost USD 0.0646139162561576/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0142856129065212/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.014285612906521182
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Qwen3.5-9B Jev-like data-mix v2",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 45.69949425428299,
    "I_sealed": 38.184197883632265,
    "base": 41.94184606895763,
    "excess": 4.94190872821248,
    "gap": 7.515296370650724,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 56.305297450788785,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 72.75675675675674,
       "hard": 49.040578798364265,
       "judge": 56.49307844429796,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 33.4120082815735,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 47.38117703048668,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.73251602617265,
       "hard": 54.043486375775075,
       "judge": 35.50551913484291,
       "standard": 50.19437568553258
      }
     },
     "sealed|choice": {
      "cc": 54.756843153610475,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 64.08247727440693,
       "hard": 46.04072398190045,
       "judge": 51.58286778398511,
       "standard": 72.28722749107035
      }
     },
     "sealed|noul": {
      "cc": 20.448763917177054,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 52,
       "hard": -4.854368932038833,
       "judge": 26.315789473684205,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 39.346986580109274,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 69.00077177598622,
       "hard": 33.3124721590493,
       "judge": 38.59270672300753,
       "standard": 37.720542609943344
      }
     }
    }
   },
   "jevbench_score": 44.38643689711108,
   "key": "qwen35-9b-jev-data-mix-v2",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (adapter code and weights; Qwen3.5-9B base is Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.908745247148289,
    "decisive_rate": 0.7088948787061995,
    "supported": true,
    "valid": 371
   },
   "open": "unknown",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.78895435560509,
     "cost": 45.690218065856605,
     "intelligence": 41.94184606895763,
     "speed": 85.40793837616167
    },
    "board_release": "v1.6.43",
    "capability": 60.86540021228136,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      22.862450507726937,
      41.74491188365231
     ],
     "B": [
      20.753639028685242,
      39.453425031751074
     ],
     "C": [
      15.87670174147704,
      28.989522141425212
     ]
    },
    "jevbench_score": 33.58966697347588,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 64,
    "rank": 94,
    "ranks": {
     "A": 94,
     "B": 93,
     "C": 96,
     "capability": 129
    },
    "scores": {
     "A": 33.58966697347588,
     "B": 31.31610274190126,
     "C": 23.326157620469353
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4865717590786517,
     "p50_s_raw": 0.16828587953932583,
     "p95_s_adjusted": 0.5916423641145229,
     "p95_s_raw": 0.22082118205726145
    },
    "views": {
     "Balanced 33:33:33": 30.68928023743452,
     "Emphasis on Accuracy 60:20:20": 27.947003437168036,
     "Emphasis on Cost 20:20:60": 29.02722060908932,
     "Emphasis on Speed 20:60:20": 36.335136837664905,
     "Intelligence only": 24.643879144700183
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 133,
   "ranked": true,
   "ranks": {
    "A": 133,
    "B": 136,
    "C": 138,
    "capability": 162
   },
   "repo": "https://huggingface.co/jsaurabh/qwen3.5-9b-jev-data-mix-v2",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 44.38643689711108,
    "B": 40.32198550559741,
    "C": 30.823914511882688
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.49227979183197024,
    "p50_s_raw": 0.1711398959159851,
    "p95_s_adjusted": 0.6102187336888164,
    "p95_s_raw": 0.23010936684440822
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.15659713819453,
      82.50836404329671
     ],
     "I": [
      35.70342834084104,
      45.9106569165651
     ],
     "capability": [
      56.2798648487553,
      63.69018289459294
     ],
     "composite": null,
     "seed": 92
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.78895435560509,
     "capability": 60.86540021228136,
     "intelligence": 41.94184606895763
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6533333333333333,
      "calibration": 71.52242653727143,
      "cc": 55.666284588350074,
      "cc_by_type": {
       "choice": 55.666284588350074
      },
      "cc_tiered": 55.53107030219963,
      "cc_tiered_open": 56.305297450788785,
      "cc_tiered_sealed": 54.756843153610475,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 55.666284588350074
     },
     "noul": {
      "accuracy": 0.6373333333333333,
      "calibration": 84.25090944976516,
      "cc": 27.46666666666666,
      "cc_by_type": {
       "noul": 27.46666666666666
      },
      "cc_tiered": 26.930386099375276,
      "cc_tiered_open": 33.4120082815735,
      "cc_tiered_sealed": 20.448763917177054,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 27.46666666666666
     },
     "score": {
      "accuracy": 0.5733333333333334,
      "calibration": 83.59352707977865,
      "cc": 42.24339448080683,
      "cc_by_type": {
       "score": 42.24339448080683
      },
      "cc_tiered": 43.36408180529798,
      "cc_tiered_open": 47.38117703048668,
      "cc_tiered_sealed": 39.346986580109274,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 42.24339448080683
     }
    },
    "raw_on_A": {
     "calibration": 80.51953554017331,
     "capability": 63.66543855723072,
     "intelligence": 46.811341574288136
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 41.49030355277687,
    "Emphasis on Accuracy 60:20:20": 35.69529742346801,
    "Emphasis on Cost 20:20:60": 43.17862757541978,
    "Emphasis on Speed 20:60:20": 47.32260485016754,
    "Intelligence only": 29.51227012683834
   }
  },
  {
   "adapter_id": "images/raw-phi-4-mini/spec.json:raw_adapter.RawControlAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Microsoft / neutral reproduction",
   "axes": {
    "calibration": 57.015465878675315,
    "cost": 53.22051180195924,
    "intelligence": 13.660237978082595,
    "speed": 91.5252083185614
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.24150351750278642,
      "ece_score": 51.69929649944272,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 51.69929649944272,
      "score_before_time_limit": 51.69929649944272,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.49595687331536387,
      "brier": 0.28688067959784064,
      "ece": 0.2513310447491633,
      "n": 371,
      "score": 49.73379105016734,
      "score_before_time_limit": 49.73379105016734,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8970189701897019,
      "interval90_width": 0.5808491418247513,
      "n": 369,
      "nrps": 0.15574884690331758,
      "score": 69.6133100864159,
      "score_before_time_limit": 69.6133100864159,
      "time_limit_failed_share": 0,
      "top_ece": 0.22599247568418224
     }
    },
    "score": 57.015465878675315
   },
   "capability": 35.33785192837895,
   "class": "raw-logit-control",
   "composite_ci95": {
    "A": [
     0.9684805743700438,
     4.125776981199927
    ],
    "B": [
     0.7191616490856636,
     3.3088114802645325
    ],
    "C": [
     0.6725559544236414,
     2.8651229036110593
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03625068965517241
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Raw Phi-4 mini direct logits",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 15.681632538277219,
    "I_sealed": 11.638843417887971,
    "base": 13.660237978082595,
    "excess": 1.469401477951004,
    "gap": 4.042789120389248,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 27.48112242812705,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 26.05405405405406,
       "hard": 16.955017301038072,
       "judge": 28.80685563612393,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -56.52173913043479,
       "judge": -50,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 19.563775186704607,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.64212383139005,
       "hard": 14.241949100287254,
       "judge": 14.5533398786592,
       "standard": 21.683905999264695
      }
     },
     "sealed|choice": {
      "cc": 25.835499598852735,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 37.47690488507872,
       "hard": 17.533936651583694,
       "judge": 26.443202979515842,
       "standard": 35.70636777928318
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 4.0000000000000036,
       "hard": -66.99029126213591,
       "judge": -34.210526315789465,
       "standard": -15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 9.081030654811178,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 50.298756504384265,
       "hard": 1.511859550820871,
       "judge": 5.6438828552519915,
       "standard": 8.766231637344024
      }
     }
    }
   },
   "jevbench_score": 2.4785276781687653,
   "key": "raw-phi-4-mini",
   "last_measured_on": "2026-10-02",
   "licence": "MIT",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6096256684491979,
    "decisive_rate": 0.5040431266846361,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 57.015465878675315,
     "cost": 53.22051180195924,
     "intelligence": 13.660237978082595,
     "speed": 91.89255362835293
    },
    "board_release": "v1.6.43",
    "capability": 35.33785192837895,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 122,
    "capability_rank": 150,
    "composite_ci95": {
     "A": [
      0.9687582607873362,
      4.12745446682704
     ],
     "B": [
      0.7192841291179243,
      3.3096762413612253
     ],
     "C": [
      0.672748792213428,
      2.866287824185444
     ]
    },
    "jevbench_score": 2.4794266829120515,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 154,
    "rank": 195,
    "ranks": {
     "A": 195,
     "B": 195,
     "C": 195,
     "capability": 216
    },
    "scores": {
     "A": 2.4794266829120515,
     "B": 1.927494035430626,
     "C": 1.721824085355591
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.23357354765757918,
     "p50_s_raw": 0.04178677382878959,
     "p95_s_adjusted": 0.27689874032512307,
     "p95_s_raw": 0.06344937016256152
    },
    "views": {
     "Balanced 33:33:33": 2.1766003082696974,
     "Emphasis on Accuracy 60:20:20": 1.497083363151444,
     "Emphasis on Cost 20:20:60": 2.657076827352907,
     "Emphasis on Speed 20:60:20": 2.994215323690922,
     "Intelligence only": 1.0196116461240732
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 235,
   "ranked": true,
   "ranks": {
    "A": 235,
    "B": 235,
    "C": 235,
    "capability": 258
   },
   "repo": "https://huggingface.co/microsoft/Phi-4-mini-instruct",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 2.4785276781687653,
    "B": 1.9270593300106893,
    "C": 1.7211997765060865
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23720237296074628,
    "p50_s_raw": 0.043601186480373144,
    "p95_s_adjusted": 0.2967290996108204,
    "p95_s_raw": 0.0733645498054102
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Phi-4-mini-instruct BF16",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      51.56094132127967,
      61.43189419566834
     ],
     "I": [
      9.603807805278013,
      16.664196620072826
     ],
     "capability": [
      31.514955827472072,
      38.142435957298304
     ],
     "composite": {
      "A": [
       0.9684805743700438,
       4.125776981199927
      ],
      "B": [
       0.7191616490856636,
       3.3088114802645325
      ],
      "C": [
       0.6725559544236414,
       2.8651229036110593
      ]
     },
     "seed": 93
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 57.015465878675315,
     "capability": 35.33785192837895,
     "intelligence": 13.660237978082595
    },
    "per_type": {
     "choice": {
      "accuracy": 0.42133333333333334,
      "calibration": 51.69929649944272,
      "cc": 25.996798120553574,
      "cc_by_type": {
       "choice": 25.996798120553574
      },
      "cc_tiered": 26.658311013489893,
      "cc_tiered_open": 27.48112242812705,
      "cc_tiered_sealed": 25.835499598852735,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 25.996798120553574
     },
     "noul": {
      "accuracy": 0.304,
      "calibration": 49.73379105016734,
      "cc": -39.2,
      "cc_by_type": {
       "noul": -39.2
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4026666666666667,
      "calibration": 69.6133100864159,
      "cc": 13.214800471337696,
      "cc_by_type": {
       "score": 13.214800471337696
      },
      "cc_tiered": 14.322402920757892,
      "cc_tiered_open": 19.563775186704607,
      "cc_tiered_sealed": 9.081030654811178,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 13.214800471337696
     }
    },
    "raw_on_A": {
     "calibration": 53.16092441580774,
     "capability": 33.04410281531054,
     "intelligence": 12.927281214813327
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 2.1756766130294327,
    "Emphasis on Accuracy 60:20:20": 1.496821108735571,
    "Emphasis on Cost 20:20:60": 2.6562508272129657,
    "Emphasis on Speed 20:60:20": 2.9910709227275794,
    "Intelligence only": 1.0196116461240732
   }
  },
  {
   "adapter_id": "images/raw-qwen3-0.6b/spec.json:raw_adapter.RawControlAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Alibaba Qwen / neutral reproduction",
   "axes": {
    "calibration": 10.94062892769364,
    "cost": 84.13747424220067,
    "intelligence": 5.554864880982027,
    "speed": 93.42809294775304
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.6186890197483605,
      "ece_score": 0,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 0,
      "score_before_time_limit": 0,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.580040467448705,
      "ece": 0.5809468363625269,
      "n": 371,
      "score": 0,
      "score_before_time_limit": 0,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.5067750677506775,
      "interval90_width": 0.31887985546522124,
      "n": 369,
      "nrps": 0.34356226433838155,
      "score": 32.82188678308092,
      "score_before_time_limit": 32.82188678308092,
      "time_limit_failed_share": 0,
      "top_ece": 0.6266298915392519
     }
    },
    "score": 10.94062892769364
   },
   "capability": 8.247746904337834,
   "class": "raw-logit-control",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.6B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 907,601 input + 0 output tokens over 1,624 decisions = 558.867610837 tokens/decision. GPU-time cost USD 0.00337870954409063/1,000; old list-price cost USD 0.00558867610837438/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00337870954409063/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003378709544090625
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Raw Qwen3 0.6B direct logits",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 6.274289147536377,
    "I_sealed": 4.835440614427676,
    "base": 5.554864880982027,
    "excess": -1.1345391093295425,
    "gap": 1.4388485331087013,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 18.822867442609134,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.945945945945947,
       "hard": 9.405473419314262,
       "judge": 28.80685563612393,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -30.434782608695656,
       "judge": -30.000000000000004,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 39.408700317953674,
       "hard": -23.213473292255358,
       "judge": -47.812096684203475,
       "standard": -39.963164880126655
      }
     },
     "sealed|choice": {
      "cc": 14.506321843283029,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 28.164954548813856,
       "hard": -1.8099547511312433,
       "judge": 26.443202979515842,
       "standard": 22.404236974996934
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -24,
       "hard": -22.330097087378643,
       "judge": -5.263157894736848,
       "standard": -1.4084507042253502
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -8.286778561594211,
       "hard": -56.317892789427006,
       "judge": -50.51708927969312,
       "standard": -75.44853557058413
      }
     }
    }
   },
   "jevbench_score": 0.1679189307325968,
   "key": "raw-qwen3-0.6b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.41778975741239893,
    "decisive_rate": 1,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 10.94062892769364,
     "cost": 77.58073177551512,
     "intelligence": 5.554864880982027,
     "speed": 93.52703208725202
    },
    "board_release": "v1.6.43",
    "capability": 8.247746904337834,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 140,
    "capability_rank": 170,
    "composite_ci95": {
     "A": [
      0.04648667178844407,
      0.37042416292233377
     ],
     "B": [
      0.033778521738855774,
      0.30178355486696584
     ],
     "C": [
      0.032282410964197275,
      0.2572390020293985
     ]
    },
    "jevbench_score": 0.1673536172919218,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 173,
    "rank": 217,
    "ranks": {
     "A": 217,
     "B": 216,
     "C": 217,
     "capability": 242
    },
    "scores": {
     "A": 0.1673536172919218,
     "B": 0.12991425235961862,
     "C": 0.11621778978605682
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2079640818759799,
     "p50_s_raw": 0.02898204093798995,
     "p95_s_adjusted": 0.21345604490488768,
     "p95_s_raw": 0.03172802245244384
    },
    "views": {
     "Balanced 33:33:33": 0.181861744313326,
     "Emphasis on Accuracy 60:20:20": 0.10948842934445303,
     "Emphasis on Cost 20:20:60": 0.26903830305770593,
     "Emphasis on Speed 20:60:20": 0.27429427173292537,
     "Intelligence only": 0.06856152826445967
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 258,
   "ranked": true,
   "ranks": {
    "A": 258,
    "B": 257,
    "C": 258,
    "capability": 283
   },
   "repo": "https://huggingface.co/Qwen/Qwen3-0.6B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.1679189307325968,
    "B": 0.13018643917289482,
    "C": 0.11661036856430335
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20884675215929746,
    "p50_s_raw": 0.029423376079648733,
    "p95_s_adjusted": 0.2174517862033099,
    "p95_s_raw": 0.03372589310165495
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-0.6B BF16",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      10.514844290675109,
      11.37896862687295
     ],
     "I": [
      3.4338072843331555,
      7.572845892568445
     ],
     "capability": [
      7.166123426414619,
      9.279855986833727
     ],
     "composite": null,
     "seed": 94
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 10.94062892769364,
     "capability": 8.247746904337834,
     "intelligence": 5.554864880982027
    },
    "per_type": {
     "choice": {
      "accuracy": 0.336,
      "calibration": 0,
      "cc": 15.0838835576859,
      "cc_by_type": {
       "choice": 15.0838835576859
      },
      "cc_tiered": 16.66459464294608,
      "cc_tiered_open": 18.822867442609134,
      "cc_tiered_sealed": 14.506321843283029,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 15.0838835576859
     },
     "noul": {
      "accuracy": 0.41333333333333333,
      "calibration": 0,
      "cc": -17.333333333333336,
      "cc_by_type": {
       "noul": -17.333333333333336
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.256,
      "calibration": 32.82188678308092,
      "cc": -46.098542643497574,
      "cc_by_type": {
       "score": -46.098542643497574
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 10.73890139109323,
     "capability": 8.515857103448049,
     "intelligence": 6.292812815802869
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.18275319972464274,
    "Emphasis on Accuracy 60:20:20": 0.10968169153575405,
    "Emphasis on Cost 20:20:60": 0.2726061308653414,
    "Emphasis on Speed 20:60:20": 0.27548260506526745,
    "Intelligence only": 0.06856152826445967
   }
  },
  {
   "adapter_id": "images/raw-qwen3-1.7b/spec.json:raw_adapter.RawControlAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Alibaba Qwen / neutral reproduction",
   "axes": {
    "calibration": 11.516326812547591,
    "cost": 84.13747424220067,
    "intelligence": 5.124286554530066,
    "speed": 93.09539316451654
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.7144612706448337,
      "ece_score": 0,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 0,
      "score_before_time_limit": 0,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.0026954177897574594,
      "brier": 0.5809971640519896,
      "ece": 0.5815255292937018,
      "n": 371,
      "score": 0,
      "score_before_time_limit": 0,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.4959349593495935,
      "interval90_width": 0.23238482384823847,
      "n": 369,
      "nrps": 0.3090203912471445,
      "score": 34.548980437642776,
      "score_before_time_limit": 34.548980437642776,
      "time_limit_failed_share": 0,
      "top_ece": 0.5704149904947305
     }
    },
    "score": 11.516326812547591
   },
   "capability": 8.320306683538828,
   "class": "raw-logit-control",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 1.7B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 907,601 input + 0 output tokens over 1,624 decisions = 558.867610837 tokens/decision. GPU-time cost USD 0.00337870954409063/1,000; old list-price cost USD 0.0111773522167488/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00337870954409063/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.003378709544090625
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Raw Qwen3 1.7B direct logits",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 7.018912428422589,
    "I_sealed": 3.2296606806375423,
    "base": 5.124286554530066,
    "excess": 1.2158641053468031,
    "gap": 3.789251747785047,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 21.05673728526777,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 41.621621621621614,
       "hard": -0.03145643284050508,
       "judge": 24.851680949241928,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -30.434782608695656,
       "judge": -30.000000000000004,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 16.019988364571248,
       "hard": -4.886927335935498,
       "judge": -32.17506026172301,
       "standard": -190.1002337898353
      }
     },
     "sealed|choice": {
      "cc": 9.688982041912627,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 21.513561451481806,
       "hard": -5.3733031674208345,
       "judge": 21.787709497206727,
       "standard": 15.753171572853812
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -24,
       "hard": -22.330097087378643,
       "judge": -5.263157894736848,
       "standard": -1.4084507042253502
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -19.19653010919571,
       "hard": -25.671423213473844,
       "judge": -56.434399118166525,
       "standard": -31.617468115507542
      }
     }
    }
   },
   "jevbench_score": 0.13792518704402346,
   "key": "raw-qwen3-1.7b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.4189189189189189,
    "decisive_rate": 0.9973045822102425,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 11.516326812547591,
     "cost": 68.54983190559568,
     "intelligence": 5.124286554530066,
     "speed": 93.23530725162705
    },
    "board_release": "v1.6.43",
    "capability": 8.320306683538828,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 139,
    "capability_rank": 169,
    "composite_ci95": {
     "A": [
      0.040682757177666354,
      0.3224989584923113
     ],
     "B": [
      0.029352394526125654,
      0.25964193163917776
     ],
     "C": [
      0.02825191470671274,
      0.22395761006410508
     ]
    },
    "jevbench_score": 0.13671939226506033,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 175,
    "rank": 219,
    "ranks": {
     "A": 219,
     "B": 218,
     "C": 219,
     "capability": 241
    },
    "scores": {
     "A": 0.13671939226506033,
     "B": 0.10452214659357986,
     "C": 0.09494402240629189
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21396372243762016,
     "p50_s_raw": 0.03198186121881008,
     "p95_s_adjusted": 0.221885605994612,
     "p95_s_raw": 0.03594280299730599
    },
    "views": {
     "Balanced 33:33:33": 0.14292662698061412,
     "Emphasis on Accuracy 60:20:20": 0.08598558171809113,
     "Emphasis on Cost 20:20:60": 0.21037073826496294,
     "Emphasis on Speed 20:60:20": 0.21708826744822254,
     "Intelligence only": 0.05382204747082655
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 260,
   "ranked": true,
   "ranks": {
    "A": 260,
    "B": 259,
    "C": 260,
    "capability": 282
   },
   "repo": "https://huggingface.co/Qwen/Qwen3-1.7B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.13792518704402346,
    "B": 0.10508401506596053,
    "C": 0.09578137989168295
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2148456271737814,
    "p50_s_raw": 0.0324228135868907,
    "p95_s_adjusted": 0.22820973400957878,
    "p95_s_raw": 0.0391048670047894
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-1.7B BF16",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      11.044864988748188,
      11.991879282296361
     ],
     "I": [
      3.263077550449943,
      7.14487317844146
     ],
     "capability": [
      7.338168404696409,
      9.339947930556471
     ],
     "composite": null,
     "seed": 95
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 11.516326812547591,
     "capability": 8.320306683538828,
     "intelligence": 5.124286554530066
    },
    "per_type": {
     "choice": {
      "accuracy": 0.30666666666666664,
      "calibration": 0,
      "cc": 11.332569176700131,
      "cc_by_type": {
       "choice": 11.332569176700131
      },
      "cc_tiered": 15.372859663590198,
      "cc_tiered_open": 21.05673728526777,
      "cc_tiered_sealed": 9.688982041912627,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 11.332569176700131
     },
     "noul": {
      "accuracy": 0.41333333333333333,
      "calibration": 0,
      "cc": -17.333333333333336,
      "cc_by_type": {
       "noul": -17.333333333333336
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.304,
      "calibration": 34.548980437642776,
      "cc": -31.4596497298246,
      "cc_by_type": {
       "score": -31.4596497298246
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 11.45520847967844,
     "capability": 9.226470190380851,
     "intelligence": 6.997731901083261
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.14468977765975075,
    "Emphasis on Accuracy 60:20:20": 0.08636546961545986,
    "Emphasis on Cost 20:20:60": 0.21741808618246522,
    "Emphasis on Speed 20:60:20": 0.21949656714755691,
    "Intelligence only": 0.05382204747082655
   }
  },
  {
   "adapter_id": "images/raw-qwen3-4b-instruct-2507/spec.json:raw_adapter.RawControlAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Alibaba Qwen / neutral reproduction",
   "axes": {
    "calibration": 35.18175191334879,
    "cost": 77.90260990054026,
    "intelligence": 33.95117742100547,
    "speed": 90.72731866330435
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.44119675391398117,
      "ece_score": 11.760649217203767,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 11.760649217203767,
      "score_before_time_limit": 11.760649217203767,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.07547169811320753,
      "brier": 0.26616192586960774,
      "ece": 0.254956633051564,
      "n": 371,
      "score": 49.008673389687196,
      "score_before_time_limit": 49.008673389687196,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.5501355013550135,
      "interval90_width": 0.07452574525745256,
      "n": 369,
      "nrps": 0.1781187945424487,
      "score": 44.77593313315542,
      "score_before_time_limit": 44.77593313315542,
      "time_limit_failed_share": 0,
      "top_ece": 0.46318127139722143
     }
    },
    "score": 35.18175191334879
   },
   "capability": 34.56646466717713,
   "class": "raw-logit-control",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 901,105 input + 0 output tokens over 1,624 decisions = 554.867610837 tokens/decision. GPU-time cost USD 0.00545229907258527/1,000; old list-price cost USD 0.0166460283251232/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00545229907258527/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005452299072585267
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Raw Qwen3 4B Instruct 2507 direct logits",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 39.33458111901565,
    "I_sealed": 28.646049395387923,
    "base": 33.990315257201786,
    "excess": 8.115144081189486,
    "gap": 10.68853172362773,
    "penalty": 0.9988485591881051,
    "per_type_split": {
     "open|choice": {
      "cc": 53.102399069057086,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 39.60364894620951,
       "judge": 44.62755438365195,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 40.36853002070393,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 19.999999999999996,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 24.532814267285957,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 44.98343409680498,
       "hard": 0.7007591330399521,
       "judge": 15.311071908696839,
       "standard": 75.80422815890212
      }
     },
     "sealed|choice": {
      "cc": 41.64735120428594,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 53.44024831867564,
       "hard": 30.2601809954751,
       "judge": 41.34078212290503,
       "standard": 58.9850966867841
      }
     },
     "sealed|noul": {
      "cc": 25.067667527906327,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 0.9708737864077666,
       "judge": 31.578947368421062,
       "standard": 38.0281690140845
      }
     },
     "sealed|score": {
      "cc": 19.223129453971513,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 48.978480535664694,
       "hard": 7.881002538538395,
       "judge": 13.283448182092084,
       "standard": 35.9392296518103
      }
     }
    }
   },
   "jevbench_score": 22.563881701127844,
   "key": "raw-qwen3-4b-instruct-2507",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7259475218658892,
    "decisive_rate": 0.9245283018867925,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 35.18175191334879,
     "cost": 63.36068112101963,
     "intelligence": 33.95117742100547,
     "speed": 91.16296741568695
    },
    "board_release": "v1.6.43",
    "capability": 34.56646466717713,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 123,
    "capability_rank": 152,
    "composite_ci95": {
     "A": [
      13.8369005003162,
      28.950164844761904
     ],
     "B": [
      12.606782226433888,
      27.192389151058098
     ],
     "C": [
      9.608958680775137,
      20.10428114219577
     ]
    },
    "jevbench_score": 21.792436038282993,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 83,
    "rank": 116,
    "ranks": {
     "A": 116,
     "B": 128,
     "C": 134,
     "capability": 219
    },
    "scores": {
     "A": 21.792436038282993,
     "B": 20.20761454160472,
     "C": 15.133636137696522
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24207517188042402,
     "p50_s_raw": 0.04603758594021201,
     "p95_s_adjusted": 0.31604797318577765,
     "p95_s_raw": 0.08302398659288883
    },
    "views": {
     "Balanced 33:33:33": 24.609772187762267,
     "Emphasis on Accuracy 60:20:20": 20.02676319678133,
     "Emphasis on Cost 20:20:60": 26.265558574780524,
     "Emphasis on Speed 20:60:20": 29.50118725011904,
     "Intelligence only": 15.653970524552685
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 180,
   "ranked": true,
   "ranks": {
    "A": 180,
    "B": 182,
    "C": 183,
    "capability": 260
   },
   "repo": "https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 22.563881701127844,
    "B": 20.733465515414572,
    "C": 15.66936229244989
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24956028824672102,
    "p50_s_raw": 0.049780144123360515,
    "p95_s_adjusted": 0.33891645497642453,
    "p95_s_raw": 0.09445822748821228
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-4B-Instruct-2507 BF16",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      30.079848447424265,
      39.87330204447962
     ],
     "I": [
      28.52380607287802,
      37.88233031513499
     ],
     "capability": [
      29.696940965810747,
      38.2358105496228
     ],
     "composite": null,
     "seed": 96
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 35.18175191334879,
     "capability": 34.56646466717713,
     "intelligence": 33.95117742100547
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5626666666666666,
      "calibration": 11.760649217203767,
      "cc": 44.07131286530316,
      "cc_by_type": {
       "choice": 44.07131286530316
      },
      "cc_tiered": 47.37487513667151,
      "cc_tiered_open": 53.102399069057086,
      "cc_tiered_sealed": 41.64735120428594,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 44.07131286530316
     },
     "noul": {
      "accuracy": 0.664,
      "calibration": 49.008673389687196,
      "cc": 32.800000000000004,
      "cc_by_type": {
       "noul": 32.800000000000004
      },
      "cc_tiered": 32.718098774305126,
      "cc_tiered_open": 40.36853002070393,
      "cc_tiered_sealed": 25.067667527906327,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 32.800000000000004
     },
     "score": {
      "accuracy": 0.456,
      "calibration": 44.77593313315542,
      "cc": 20.99121192415334,
      "cc_by_type": {
       "score": 20.99121192415334
      },
      "cc_tiered": 21.877971860628733,
      "cc_tiered_open": 24.532814267285957,
      "cc_tiered_sealed": 19.223129453971513,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 20.99121192415334
     }
    },
    "raw_on_A": {
     "calibration": 37.75992930947443,
     "capability": 38.17464116904609,
     "intelligence": 38.58935302861775
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 25.94542267802686,
    "Emphasis on Accuracy 60:20:20": 20.543123654997363,
    "Emphasis on Cost 20:20:60": 29.187110261292926,
    "Emphasis on Speed 20:60:20": 30.592693473748177,
    "Intelligence only": 15.653970524552685
   }
  },
  {
   "adapter_id": "images/raw-qwen3-8b/spec.json:raw_adapter.RawControlAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Alibaba Qwen / neutral reproduction",
   "axes": {
    "calibration": 33.721548452619835,
    "cost": 68.77812304352645,
    "intelligence": 26.54912920590792,
    "speed": 88.245839907826
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.43645983930453747,
      "ece_score": 12.708032139092506,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 12.708032139092506,
      "score_before_time_limit": 12.708032139092506,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.07816711590296499,
      "brier": 0.2740805017981965,
      "ece": 0.2700267098593338,
      "n": 371,
      "score": 45.99465802813324,
      "score_before_time_limit": 45.99465802813324,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.5474254742547425,
      "interval90_width": 0.14972899728997294,
      "n": 369,
      "nrps": 0.21398318753780107,
      "score": 42.461955190633766,
      "score_before_time_limit": 42.461955190633766,
      "time_limit_failed_share": 0,
      "top_ece": 0.46838885432476185
     }
    },
    "score": 33.721548452619835
   },
   "capability": 30.13533882926388,
   "class": "raw-logit-control",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 8B active parameters; throughput 30762.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 907,601 input + 0 output tokens over 1,624 decisions = 558.867610837 tokens/decision. GPU-time cost USD 0.0109832085951747/1,000; old list-price cost USD 0.0653875104679803/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0109832085951747/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.010983208595174725
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Raw Qwen3 8B direct logits",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 31.420181182575643,
    "I_sealed": 21.6780772292402,
    "base": 26.54912920590792,
    "excess": 7.168716310897201,
    "gap": 9.742103953335445,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 51.791553839411165,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 33.941491034916645,
       "judge": 60.44825313117996,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 34.967563837129056,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": 39.99999999999999,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 7.50142587118671,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 42.05293172396295,
       "hard": 3.1607638000191285,
       "judge": 1.3576889332232334,
       "standard": 8.122602494078969
      }
     },
     "sealed|choice": {
      "cc": 41.65523094061268,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 57.43108417707487,
       "hard": 31.787330316742068,
       "judge": 39.47858472998139,
       "standard": 56.76807488606972
      }
     },
     "sealed|noul": {
      "cc": 22.61439253816204,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 60.00000000000001,
       "hard": -6.796116504854366,
       "judge": 31.578947368421062,
       "standard": 49.29577464788733
      }
     },
     "sealed|score": {
      "cc": 0.7646082089458659,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 50.377225661492794,
       "hard": -19.926430375972124,
       "judge": 5.951813095928216,
       "standard": 9.559569322034855
      }
     }
    }
   },
   "jevbench_score": 12.101558108732878,
   "key": "raw-qwen3-8b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7105263157894737,
    "decisive_rate": 0.921832884097035,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 33.721548452619835,
     "cost": 45.53515592313027,
     "intelligence": 26.54912920590792,
     "speed": 88.65282990878164
    },
    "board_release": "v1.6.43",
    "capability": 30.13533882926388,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      5.526896473528715,
      13.605697544750722
     ],
     "B": [
      4.901937683715855,
      12.59857128804635
     ],
     "C": [
      3.838122551061608,
      9.448401072743554
     ]
    },
    "jevbench_score": 9.30125972789427,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 135,
    "rank": 174,
    "ranks": {
     "A": 174,
     "B": 172,
     "C": 175,
     "capability": 229
    },
    "scores": {
     "A": 9.30125972789427,
     "B": 8.458419096926727,
     "C": 6.4592081443710185
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.30506142796948554,
     "p50_s_raw": 0.07753071398474276,
     "p95_s_adjusted": 0.4470228368416429,
     "p95_s_raw": 0.14851141842082144
    },
    "views": {
     "Balanced 33:33:33": 9.893413746086136,
     "Emphasis on Accuracy 60:20:20": 7.995042950818957,
     "Emphasis on Cost 20:20:60": 10.181976971451938,
     "Emphasis on Speed 20:60:20": 12.509103386849494,
     "Intelligence only": 6.208182595906101
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 212,
   "ranked": true,
   "ranks": {
    "A": 212,
    "B": 212,
    "C": 212,
    "capability": 270
   },
   "repo": "https://huggingface.co/Qwen/Qwen3-8B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 12.101558108732878,
    "B": 10.772829193838941,
    "C": 8.403859797731164
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.31334649920463564,
    "p50_s_raw": 0.08167324960231781,
    "p95_s_adjusted": 0.47795957052148874,
    "p95_s_raw": 0.16397978526074436
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-8B BF16",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      28.041560709983848,
      38.19473887695128
     ],
     "I": [
      21.71577559015635,
      30.903657696346237
     ],
     "capability": [
      25.49211567892071,
      33.98867030681561
     ],
     "composite": null,
     "seed": 97
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 33.721548452619835,
     "capability": 30.13533882926388,
     "intelligence": 26.54912920590792
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5613333333333334,
      "calibration": 12.708032139092506,
      "cc": 43.90079857525836,
      "cc_by_type": {
       "choice": 43.90079857525836
      },
      "cc_tiered": 46.723392390011924,
      "cc_tiered_open": 51.791553839411165,
      "cc_tiered_sealed": 41.65523094061268,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 43.90079857525836
     },
     "noul": {
      "accuracy": 0.648,
      "calibration": 45.99465802813324,
      "cc": 29.600000000000005,
      "cc_by_type": {
       "noul": 29.600000000000005
      },
      "cc_tiered": 28.79097818764555,
      "cc_tiered_open": 34.967563837129056,
      "cc_tiered_sealed": 22.61439253816204,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 29.600000000000005
     },
     "score": {
      "accuracy": 0.3973333333333333,
      "calibration": 42.461955190633766,
      "cc": 3.4025581754630996,
      "cc_by_type": {
       "score": 3.4025581754630996
      },
      "cc_tiered": 4.133017040066288,
      "cc_tiered_open": 7.50142587118671,
      "cc_tiered_sealed": 0.7646082089458659,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 3.4025581754630996
     }
    },
    "raw_on_A": {
     "calibration": 38.60246474366818,
     "capability": 35.16687873306722,
     "intelligence": 31.73129272246627
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 13.31225576850268,
    "Emphasis on Accuracy 60:20:20": 10.151344199264557,
    "Emphasis on Cost 20:20:60": 15.22096212523977,
    "Emphasis on Speed 20:60:20": 16.353696099883134,
    "Intelligence only": 7.485327984239622
   }
  },
  {
   "adapter_id": "images/reflex-4b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "kshetrajna12",
   "axes": {
    "calibration": 86.9025081232179,
    "cost": 77.6846417557845,
    "intelligence": 30.80973256512164,
    "speed": 68.75878067126158
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.068294434375,
      "ece_score": 86.341113125,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 86.11087015666666,
      "score_before_time_limit": 86.341113125,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "noul": {
      "abstention_rate": 0.41866666666666663,
      "brier": 0.14083422852570657,
      "ece": 0.05314781866666669,
      "n": 375,
      "score": 89.37043626666666,
      "score_before_time_limit": 89.37043626666666,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9866666666666667,
      "interval90_width": 0.7284444444444443,
      "n": 375,
      "nrps": 0.1108956250735923,
      "score": 85.22621794632039,
      "score_before_time_limit": 85.22621794632039,
      "time_limit_failed_share": 0,
      "top_ece": 0.09229000799999995
     }
    },
    "score": 86.9025081232179
   },
   "capability": 58.85612034416977,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 916,307 input + 0 output tokens over 1,624 decisions = 564.228448276 tokens/decision. GPU-time cost USD 0.00554428152801659/1,000; old list-price cost USD 0.0169268534482759/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00554428152801659/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005544281528016589
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 2,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "reflex 4B (kshetrajna12)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 33.07061549561215,
    "I_sealed": 28.548849634631125,
    "base": 30.80973256512164,
    "excess": 1.9483782185427834,
    "gap": 4.521765860981027,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 54.32138182784878,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 49.040578798364265,
       "judge": 48.58272907053395,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 10.45962732919255,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 30.434782608695656,
       "judge": -19.999999999999996,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 34.430837329795125,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.20111818181819,
       "hard": 24.787801851851842,
       "judge": 33.86929491525422,
       "standard": 45.67408148148149
      }
     },
     "sealed|choice": {
      "cc": 53.57907452177943,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 50.113122171945705,
       "judge": 50.65176908752329,
       "standard": 58.9850966867841
      }
     },
     "sealed|noul": {
      "cc": 1.7255356358899414,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 39.99999999999999,
       "hard": -14.563106796116509,
       "judge": 5.263157894736836,
       "standard": 9.859154929577475
      }
     },
     "sealed|score": {
      "cc": 30.341938746224002,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 54.91348368794326,
       "hard": 26.727804411764712,
       "judge": 23.787971856287427,
       "standard": 35.11538527918782
      }
     }
    }
   },
   "jevbench_score": 21.277263784731073,
   "key": "reflex-4b",
   "last_measured_on": "2026-10-02",
   "licence": "MIT (code, adapter); Apache-2.0 (base)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9174311926605505,
    "decisive_rate": 0.5813333333333334,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.97925577932901,
     "cost": 63.142712976263866,
     "intelligence": 30.935558296483705,
     "speed": 68.7900017304639
    },
    "board_release": "v1.6.43",
    "capability": 58.95740703790636,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      14.765436275328822,
      29.209404384423753
     ],
     "B": [
      12.581433816044255,
      26.114502689493058
     ],
     "C": [
      10.253775191200571,
      20.28430860029427
     ]
    },
    "jevbench_score": 20.637311182702398,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 92,
    "rank": 126,
    "ranks": {
     "A": 126,
     "B": 137,
     "C": 138,
     "capability": 138
    },
    "scores": {
     "A": 20.637311182702398,
     "B": 17.968342404009285,
     "C": 14.33146609909889
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 2.865421538101509,
     "p50_s_raw": 1.3577107690507546,
     "p95_s_adjusted": 4.611171829840169,
     "p95_s_raw": 2.230585914920084
    },
    "views": {
     "Balanced 33:33:33": 18.316133226478374,
     "Emphasis on Accuracy 60:20:20": 15.029593546249647,
     "Emphasis on Cost 20:20:60": 20.28126457022741,
     "Emphasis on Speed 20:60:20": 20.855909642703498,
     "Intelligence only": 11.842240202134587
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 184,
   "ranked": true,
   "ranks": {
    "A": 184,
    "B": 184,
    "C": 186,
    "capability": 171
   },
   "repo": "https://github.com/kshetrajna12/reflex",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.277263784731073,
    "B": 18.283113135220113,
    "C": 14.775877628285471
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 2.8740736179519444,
    "p50_s_raw": 1.3620368089759722,
    "p95_s_adjusted": 4.630459016701207,
    "p95_s_raw": 2.240229508350603
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B + kshetrajna12/reflex-qwen3.5-4b-lora",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.4557796549836,
      87.34421355395163
     ],
     "I": [
      26.928642054525277,
      35.778887054174305
     ],
     "capability": [
      54.73340669244083,
      60.40734224969147
     ],
     "composite": null,
     "seed": 98
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.9025081232179,
     "capability": 58.85612034416977,
     "intelligence": 30.80973256512164
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6466666666666666,
      "calibration": 86.11087015666666,
      "cc": 54.813713138126026,
      "cc_by_type": {
       "choice": 54.813713138126026
      },
      "cc_tiered": 53.95022817481411,
      "cc_tiered_open": 54.32138182784878,
      "cc_tiered_sealed": 53.57907452177943,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 54.813713138126026
     },
     "noul": {
      "accuracy": 0.5333333333333333,
      "calibration": 89.37043626666666,
      "cc": 6.666666666666665,
      "cc_by_type": {
       "noul": 6.666666666666665
      },
      "cc_tiered": 6.092581482541245,
      "cc_tiered_open": 10.45962732919255,
      "cc_tiered_sealed": 1.7255356358899414,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 6.666666666666665
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 85.22621794632039,
      "cc": 32.60462780701755,
      "cc_by_type": {
       "score": 32.60462780701755
      },
      "cc_tiered": 32.386388038009564,
      "cc_tiered_open": 34.430837329795125,
      "cc_tiered_sealed": 30.341938746224002,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 32.60462780701755
     }
    },
    "raw_on_A": {
     "calibration": 81.34463076766183,
     "capability": 58.29083157222991,
     "intelligence": 35.237032376798
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.002666666666666706,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 19.024925323542874,
    "Emphasis on Accuracy 60:20:20": 15.21363833492755,
    "Emphasis on Cost 20:20:60": 22.17369192273624,
    "Emphasis on Speed 20:60:20": 21.34066896441514,
    "Intelligence only": 11.698327542016717
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Atlas AI / @leemadov (dr.gaming127@gmail.com)",
   "axes": {
    "calibration": 73.15265267444973,
    "cost": 86.36009990882481,
    "intelligence": 3.96006223937058,
    "speed": 58.9717141639196
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14053714285714305,
      "ece_score": 71.8925714285714,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 71.8925714285714
     },
     "noul": {
      "abstention_rate": 0.9813333333333333,
      "brier": 0.2615201506933332,
      "ece": 0.1349863999999998,
      "n": 375,
      "score": 73.00272000000004
     },
     "score": {
      "interval90_coverage": 1,
      "interval90_width": 0.9808888888888888,
      "n": 375,
      "nrps": 0.19333013477111102,
      "score": 74.56266659477775,
      "top_ece": 0.15770826666666693
     }
    },
    "score": 73.15265267444973
   },
   "capability": 38.556357456910156,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.6B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 699,758 input + 0 output tokens over 1,485 decisions = 471.217508418 tokens/decision. GPU-time cost USD 0.00284880902410347/1,000; old list-price cost USD 0.00471217508417508/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00284880902410347/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.006301273333333334
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.002848809024103469
   },
   "decision_limit": null,
   "display": "Reflex Instinct 0.6B (Atlas AI)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Sandy, the evaluator's own shared CPU host (AMD Ryzen 5 3600, 6 cores / 12 threads, Zen 2, AVX2), NOT a dedicated machine: other jobs ran beside it and the 1-minute load average was logged every 60 s into logs/CONDITIONS-<key>.jsonl. The author's own serve.py, shipped in their weights repo, run with their own documented line `python serve.py --port 11472 --device cpu --repo <the pinned local snapshot directory>` inside a loopback-only network namespace with an outbound probe that failed as required, no credentials in the environment, offline (HF_HUB_OFFLINE=1). float32 on CPU, which is their own code's choice for CPU and the right one on Zen 2 (AVX2 only, where bf16 is about 5x slower). Their serve.py sets no torch thread count, so their own default is every core of the host; this run bounded it to 6 of Sandy's 12 (OMP_NUM_THREADS=6) to stay polite to the other jobs on the box - OUR cap, not theirs, and an unbounded host would use all 12. Peak RSS about 7.6 GB, so the row ran in a 14 GiB systemd scope rather than the initial 4 GiB allocation. Speed is therefore a loaded-shared-CPU figure and a lower bound for this model on a dedicated core or a GPU; the standard self-hosted x2 + 0.15 s adjustment is applied on top and the raw p50/p95 are kept. Re-measured for this release on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b) under the pre-registration, at USD 0 on Sandy’s own CPU, 6 of 12 threads, starting 2026-10-13 06:14:53Z; the 1-minute load average is logged for the whole window and reported with the row. DISCLOSED: the author’s own encode() truncates whole state lines above its own max_len=16384 rather than refusing them, so this row answers the draw’s 15 items of 106,804–122,936 characters that four other rows of this draw recorded as context refusals (attest-31b-v1.0, julia-1, standard_one_3b_sh and uniphore-4b-decider; krino-2b, both MeTask rows and torchcast-decision-4b-v1.1 answered them) — the author’s design choice, not ours, and those items are the slowest in the row.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 6.7839203524836424,
    "I_sealed": 1.136204126257517,
    "base": 3.96006223937058,
    "excess": -4.191810544129586,
    "gap": 5.647716226226125,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 20.35176105745093,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 18.27027027027028,
       "hard": 13.180245360176164,
       "judge": 32.76203032300594,
       "standard": 17.12013394725827
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -100,
       "judge": -100,
       "standard": -88.88888888888889
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 15.23909090909089,
       "hard": -5.91657407407411,
       "judge": -12.511694915254235,
       "standard": -44.39370370370372
      }
     },
     "sealed|choice": {
      "cc": 3.4086123787725513,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 25.66709021601017,
       "hard": 1.6421880420955102,
       "judge": 0.39177277179238174,
       "standard": 0.33748154397807795
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -93.10344827586206,
       "hard": -98.18181818181819,
       "judge": -100,
       "standard": -97.01492537313433
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 0.7850877192982497,
       "hard": -17.508647887323825,
       "judge": -3.979895833333358,
       "standard": -4.642090909091023
      }
     }
    }
   },
   "jevbench_score": 0.08513390654661894,
   "key": "reflex-instinct-0.6b",
   "last_measured_on": "2026-10-13",
   "licence": "Apache-2.0 (model card; base Qwen/Qwen3-0.6B Apache-2.0)",
   "listing": "remeasure-pending",
   "measured_in": "v1.6.15",
   "measurement_date_status": "measured",
   "method": "v1.6",
   "model_pin": "Atlas-AI-research/reflex-instinct-0.6b@ae04b843e7840d07776c3dc554b4cd064f66cbe5",
   "not_ranked_because": "Re-measure pending under the 10 s rule: this row's per-decision times were recorded on a loaded CPU host, so they cannot be scored under the time limit. Shown with its v1.6 values until it is measured again.",
   "noul_decisive": {
    "acc_among_decisive": 0.8571428571428571,
    "decisive": 7,
    "decisive_rate": 0.018666666666666668,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 73.15265267444973,
     "cost": 79.80335744213926,
     "intelligence": 3.96006223937058,
     "speed": 58.9717141639196
    },
    "board_release": "v1.6.43",
    "capability": 38.556357456910156,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.010780726312872546,
      0.26774780433532175
     ],
     "B": [
      0.007010377455719735,
      0.18643904716544285
     ],
     "C": [
      0.00748661549505038,
      0.18593597523286234
     ]
    },
    "jevbench_score": 0.08485997925452016,
    "measured_in": "v1.6.15",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": 0.08485997925452016,
     "B": 0.05721296264713309,
     "C": 0.05893054114897234
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 3.894755522988271,
     "p50_s_raw": 1.8723777614941355,
     "p95_s_adjusted": 32.53481801018057,
     "p95_s_raw": 16.192409005090287
    },
    "views": {
     "Balanced 33:33:33": 0.06673009386029828,
     "Emphasis on Accuracy 60:20:20": 0.039850211293521905,
     "Emphasis on Cost 20:20:60": 0.10213986024727835,
     "Emphasis on Speed 20:60:20": 0.09927765123489675,
     "Intelligence only": 0.02484082563390453
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/Atlas-AI-research/reflex-instinct-0.6b",
   "scores": {
    "A": 0.08513390654661894,
    "B": 0.05731242598675418,
    "C": 0.059120768435152043
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "n": 124,
    "p50_s_adjusted": 3.894755522988271,
    "p50_s_raw": 1.8723777614941355,
    "p95_s_adjusted": 32.53481801018057,
    "p95_s_raw": 16.192409005090287
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.22727272727272727,
       "cc": -49.1,
       "cc_by_type": {
        "choice": 0,
        "noul": -100,
        "score": -47.30000000000001
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.27722772277227725,
       "cc": -29.19681711735595,
       "cc_by_type": {
        "choice": 14.103923647932142,
        "noul": -100,
        "score": -1.6943749999999813
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.19801980198019803,
       "cc": -43.559006670835934,
       "cc_by_type": {
        "choice": 5.691056910569103,
        "noul": -100,
        "score": -36.368076923076906
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.125,
       "cc": -31.862645502645506,
       "cc_by_type": {
        "choice": -14.285714285714285,
        "noul": -100,
        "score": 18.69777777777777
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.2222222222222222,
       "cc": -30.8278251460995,
       "cc_by_type": {
        "choice": 1.8286814244466008,
        "noul": -100,
        "score": 5.6878431372549
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.2779552715654952,
       "cc": -22.6697187396271,
       "cc_by_type": {
        "choice": 26.300959760500138,
        "noul": -87.62886597938144,
        "score": -6.681249999999994
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -53.91467741935484,
       "cc_by_type": {
        "choice": -16.129032258064516,
        "noul": -100,
        "score": -45.61500000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -34.40829978726886,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100,
        "score": -28.998095238095246
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.2,
       "cc": -36.25301846655777,
       "cc_by_type": {
        "choice": 1.2063906097163464,
        "noul": -100,
        "score": -9.965446009389645
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.12790697674418605,
       "cc": -57.09688888888889,
       "cc_by_type": {
        "choice": -6.666666666666665,
        "noul": -100,
        "score": -64.62400000000001
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.1875,
       "cc": -3.861517976031954,
       "cc_by_type": {
        "choice": -3.861517976031954
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 0
      },
      "perturb_pair": {
       "accuracy": 0.28125,
       "cc": -41.36431007931008,
       "cc_by_type": {
        "choice": 15.91591591591592,
        "noul": -100,
        "score": -40.00884615384614
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.16363636363636364,
       "cc": -37.397368888888884,
       "cc_by_type": {
        "choice": -4.166666666666629,
        "noul": -100,
        "score": -8.025440000000028
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -34.650927960927966,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": -100,
        "score": -27.762307692307697
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.14285714285714285,
       "cc": -52.631578947368425,
       "cc_by_type": {
        "choice": -5.263157894736845,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.15384615384615385,
       "cc": -36.06443625192013,
       "cc_by_type": {
        "choice": 17.050691244239633,
        "noul": -100,
        "score": -25.24400000000002
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.34375,
       "cc": -14.144529947415672,
       "cc_by_type": {
        "choice": 28.318584070796458,
        "noul": -75,
        "score": 4.247826086956518
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "el": {
       "accuracy": 0.2,
       "cc": -25.974627560521416,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100,
        "score": 5.874999999999986
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.21025195482189402,
       "cc": -33.525042776558344,
       "cc_by_type": {
        "choice": 5.761462515712473,
        "noul": -97.84172661870504,
        "score": -8.494864226682463
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.3870967741935484,
       "cc": -26.126930556888343,
       "cc_by_type": {
        "choice": 31.484502446982063,
        "noul": -100,
        "score": -9.865294117647094
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0,
       "cc": -45.96251366120219,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100,
        "score": -6.73999999999999
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.2857142857142857,
       "cc": -35.1146439628483,
       "cc_by_type": {
        "choice": -5.882352941176468,
        "noul": -100,
        "score": 0.538421052631588
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.23809523809523808,
       "cc": -40.884627560521416,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100,
        "score": -38.85500000000002
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0,
       "cc": -66.66666666666667,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.3333333333333333,
       "cc": -29.927109824444205,
       "cc_by_type": {
        "choice": 43.88152766952456,
        "noul": -100,
        "score": -33.66285714285717
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.24,
       "cc": -30.554920483628976,
       "cc_by_type": {
        "choice": -0.07942811755361648,
        "noul": -80,
        "score": -11.585333333333313
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -32.21272486772487,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": -7.749285714285725
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.14285714285714285,
       "cc": -33.672222222222224,
       "cc_by_type": {
        "choice": 0,
        "noul": -100,
        "score": -1.0166666666666657
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0,
       "cc": -47.50337662337663,
       "cc_by_type": {
        "choice": -29.87012987012987,
        "noul": -100,
        "score": -12.640000000000029
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.3076923076923077,
       "cc": -20.02176162216602,
       "cc_by_type": {
        "choice": 23.82564536606009,
        "noul": -80,
        "score": -3.890930232558154
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.17857142857142858,
       "cc": -40.33124267366193,
       "cc_by_type": {
        "choice": -6.069534472598708,
        "noul": -100,
        "score": -14.924193548387098
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0.09090909090909091,
       "cc": -30.057553191489365,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -100,
        "score": 11.954999999999982
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.15384615384615385,
       "cc": -55.03855072463768,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": -100,
        "score": -56.42
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.14285714285714285,
       "cc": -31.236397849462367,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100,
        "score": 3.0649999999999844
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.2,
       "cc": -43.39039548022598,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100,
        "score": -53.90000000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.2786666666666667,
       "cc": 6.829445645726105,
       "cc_by_type": {
        "choice": 6.829445645726105
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 6.829445645726105
      },
      "noul": {
       "accuracy": 0.016,
       "cc": -96.8,
       "cc_by_type": {
        "noul": -96.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.2906666666666667,
       "cc": -9.011182310469268,
       "cc_by_type": {
        "score": -9.011182310469268
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.13636363636363635,
       "cc": -45.081568627451,
       "cc_by_type": {
        "choice": -12,
        "noul": -100,
        "score": -23.24470588235299
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.08333333333333333,
       "cc": -38.55361920688007,
       "cc_by_type": {
        "choice": -11.801242236024848,
        "noul": -100,
        "score": -3.859615384615367
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.21428571428571427,
       "cc": -13.407046070460707,
       "cc_by_type": {
        "choice": -13.821138211382115,
        "noul": -100,
        "score": 73.6
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -32.693250487329436,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100,
        "score": -4.512500000000008
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.11764705882352941,
       "cc": -37.33192632850242,
       "cc_by_type": {
        "choice": -15.94202898550725,
        "noul": -100,
        "score": 3.9462499999999956
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.15789473684210525,
       "cc": -40.67872117400419,
       "cc_by_type": {
        "choice": -25.786163522012576,
        "noul": -100,
        "score": 3.75000000000002
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.23333333333333334,
       "cc": -25.75370537958774,
       "cc_by_type": {
        "choice": 21.568627450980394,
        "noul": -100,
        "score": 1.1702564102563873
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.18181818181818182,
       "cc": -33.214854315002775,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100,
        "score": -16.0208695652174
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.06666666666666667,
       "cc": -41.89425925925925,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -100,
        "score": -21.51611111111109
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.13636363636363635,
       "cc": -42.833514697294135,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -100,
        "score": -25.71681818181817
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.12,
       "cc": -42.82802325581395,
       "cc_by_type": {
        "choice": -16.279069767441868,
        "noul": -100,
        "score": -12.204999999999998
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -28.936626344086022,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -22.293750000000003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.2222222222222222,
       "cc": -36.136191263240214,
       "cc_by_type": {
        "choice": 7.869481765834927,
        "noul": -100,
        "score": -16.27805555555557
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.31797235023041476,
       "cc": -17.510075757575745,
       "cc_by_type": {
        "choice": 34.54545454545457,
        "noul": -81.81818181818181,
        "score": -5.257499999999982
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0
      },
      "moderation_trust_safety": {
       "accuracy": 0.2222222222222222,
       "cc": -37.63747958300402,
       "cc_by_type": {
        "choice": 19.385796545105567,
        "noul": -100,
        "score": -32.29823529411764
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.05555555555555555,
       "cc": -45.341573459038635,
       "cc_by_type": {
        "choice": -16.991643454038996,
        "noul": -100,
        "score": -19.033076923076898
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.16666666666666666,
       "cc": -37.452592485269975,
       "cc_by_type": {
        "choice": -18.226600985221676,
        "noul": -100,
        "score": 5.868823529411749
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.17391304347826086,
       "cc": -39.18167632850242,
       "cc_by_type": {
        "choice": -15.942028985507239,
        "noul": -100,
        "score": -1.603000000000021
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.08333333333333333,
       "cc": -38.66853905225698,
       "cc_by_type": {
        "choice": -17.43772241992883,
        "noul": -100,
        "score": 1.43210526315789
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.93670989622872,
      76.69302559358876
     ],
     "I": [
      1.940386716570835,
      5.948109684014943
     ],
     "capability": [
      35.650836259370074,
      40.70839403134445
     ],
     "composite": null,
     "seed": 63
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.15265267444973,
     "capability": 38.556357456910156,
     "intelligence": 3.96006223937058
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2786666666666667,
      "calibration": 71.8925714285714,
      "cc": 6.829445645726105,
      "cc_by_type": {
       "choice": 6.829445645726105
      },
      "cc_tiered": 11.88018671811174,
      "cc_tiered_open": 20.35176105745093,
      "cc_tiered_sealed": 3.4086123787725513,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 6.829445645726105
     },
     "noul": {
      "accuracy": 0.016,
      "calibration": 73.00272000000004,
      "cc": -96.8,
      "cc_by_type": {
       "noul": -96.8
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2906666666666667,
      "calibration": 74.56266659477775,
      "cc": -9.011182310469268,
      "cc_by_type": {
       "score": -9.011182310469268
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 74.53494666952643,
     "capability": 39.32102819546208,
     "intelligence": 4.107109721397716
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.0669559748232002,
    "Emphasis on Accuracy 60:20:20": 0.039898440082000636,
    "Emphasis on Cost 20:20:60": 0.10309812850714123,
    "Emphasis on Speed 20:60:20": 0.09957752095311366,
    "Intelligence only": 0.02484082563390453
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Elon Salfati / salfatigroup",
   "axes": {
    "calibration": 90.18677086647014,
    "cost": 51.285700298534536,
    "intelligence": 61.72139577013068,
    "speed": 86.74281071257914
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.043257328990227974,
      "ece_score": 91.34853420195441,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 91.34853420195441,
      "score_before_time_limit": 91.34853420195441,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.22102425876010778,
      "brier": 0.06278221024258757,
      "ece": 0.05196765498652292,
      "n": 371,
      "score": 89.60646900269542,
      "score_before_time_limit": 89.60646900269542,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.986449864498645,
      "interval90_width": 0.3859530261969287,
      "n": 369,
      "nrps": 0.06312416440831078,
      "score": 89.60530939476061,
      "score_before_time_limit": 89.60530939476061,
      "time_limit_failed_share": 0,
      "top_ece": 0.07238482384823842
     }
    },
    "score": 90.18677086647014
   },
   "capability": 75.9540833183004,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,038,365 input + 0 output tokens over 1,477 decisions = 703.023019634 tokens/decision. GPU-time cost USD 0.042054259482183/1,000; old list-price cost USD 0.0632720717670955/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.042054259482183/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.062301899999999986
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.042054259482183
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "René-1 31B FP8 (salfatigroup, Gemma 4 31B full fine-tune, one-pass option readout)",
   "endpoint_condition": "Candidate re-measured on 2026-10-07, current v1.6.0/v1.6.1 1500-item pool (sha256901983ae...). Immutable salfatigroup/rene-1-31b-fp8@bd634489957f8da63ccce858ee33fd6c2929d286, Apache-2.0. Same unmodified reviewed contributor source and run-39 transport shim. Author noul/yesno type mapping only; two-decimal probability rounding retained. One evaluator-owned RTX PRO6000 Blackwell 96GB pod, loopback-only network namespace, offline credential-free inference; no external hosted API, no retry or batching. Unmodified run_v16/typesafe adapter, serial one request at a time, selfhosted x2+0.15s latency adjustment. Missing ATTRIBUTIONS.md and NOTICE disclosed: training-data overlap cannot be checked. The earlier attribution hold was resolved after Apache-2.0 licensing was confirmed. COMPLETE: 1500/1500 distinct input IDs, 1477 HTTP200 native answers,23 HTTP422 refusals under the author's own32768-token state+question /65536-token request budget; every refusal counted wrong. Smoke passed all8 checks. All42 release-manifest files and6 prior source files hash-verified. Raw sha256 a28a8f0068f38b548f064a6e678bf63b13304f9add0a67b448dcc8003c9f38f1. Stack torch2.14.0, transformers5.17.0, compressed-tensors0.19.0, accelerate1.15.0. Provider pod91916684-9980-419c-98b7-c7a714a3912c removed and verified absent; approxUSD0.17. Fresh latest feed places this candidate inside both official caps.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO6000 Blackwell",
   "intelligence": {
    "I_open": 62.91896990676746,
    "I_sealed": 60.523821633493895,
    "base": 61.72139577013068,
    "excess": -0.17823936916468108,
    "gap": 2.3951482732735627,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 78.65680231931067,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396666,
       "judge": 84.17930125247197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 53.325051759834366,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 56.77505564115734,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.5151515151515,
       "hard": 65.03703703703704,
       "judge": 54.15254237288135,
       "standard": 41.8148148148148
      }
     },
     "sealed|choice": {
      "cc": 78.92231555231959,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 71.49321266968325,
       "judge": 86.96461824953444,
       "standard": 81.15531469392782
      }
     },
     "sealed|noul": {
      "cc": 47.36482975523042,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 80,
       "hard": 32.038834951456316,
       "judge": 50,
       "standard": 57.746478873239425
      }
     },
     "sealed|score": {
      "cc": 55.28431959293169,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 68.97163120567376,
       "hard": 50.97549019607843,
       "judge": 64.15568862275447,
       "standard": 43.75126903553299
      }
     }
    }
   },
   "jevbench_score": 68.59077119594947,
   "key": "rene-1-31b-fp8",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "salfatigroup/rene-1-31b-fp8@bd634489957f8da63ccce858ee33fd6c2929d286",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9826989619377162,
    "decisive_rate": 0.7789757412398922,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.18677086647014,
     "cost": 45.963638345890566,
     "intelligence": 61.72139577013068,
     "speed": 87.11720932683018
    },
    "board_release": "v1.6.43",
    "capability": 75.9540833183004,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 12,
    "capability_rank": 22,
    "composite_ci95": {
     "A": [
      54.25035390810508,
      56.621792265328125
     ],
     "B": [
      52.791631605081,
      56.42470571115517
     ],
     "C": [
      47.82528165394186,
      56.621792265328125
     ]
    },
    "jevbench_score": 55.84876743320955,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 36,
    "rank": 60,
    "ranks": {
     "A": 60,
     "B": 59,
     "C": 47,
     "capability": 40
    },
    "scores": {
     "A": 55.84876743320955,
     "B": 55.06950723833363,
     "C": 55.84876743320955
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3694145163521171,
     "p50_s_raw": 0.10970725817605853,
     "p95_s_adjusted": 0.5257328446954489,
     "p95_s_raw": 0.18786642234772444
    },
    "views": {
     "Balanced 33:33:33": 51.281219378577866,
     "Emphasis on Accuracy 60:20:20": 51.62853971578034,
     "Emphasis on Cost 20:20:60": 45.45809395907335,
     "Emphasis on Speed 20:60:20": 58.36507393476142,
     "Intelligence only": 52.15843207508249
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 90,
   "ranked": true,
   "ranks": {
    "A": 90,
    "B": 78,
    "C": 64,
    "capability": 58
   },
   "repo": "https://huggingface.co/salfatigroup/rene-1-31b-fp8",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.59077119594947,
    "B": 67.09723371935465,
    "C": 68.59077119594947
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.37661420051008465,
    "p50_s_raw": 0.11330710025504231,
    "p95_s_adjusted": 0.5621111983433366,
    "p95_s_raw": 0.20605559917166827
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-31B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.96895555807659,
      90.68721224415906
     ],
     "I": [
      56.323580074482344,
      65.99929373375565
     ],
     "capability": [
      72.10629528396096,
      77.53024725815187
     ],
     "composite": null,
     "seed": 128
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.18677086647014,
     "capability": 75.9540833183004,
     "intelligence": 61.72139577013068
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8346666666666667,
      "calibration": 91.34853420195441,
      "cc": 78.85622803444389,
      "cc_by_type": {
       "choice": 78.85622803444389
      },
      "cc_tiered": 78.78955893581514,
      "cc_tiered_open": 78.65680231931067,
      "cc_tiered_sealed": 78.92231555231959,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 78.85622803444389
     },
     "noul": {
      "accuracy": 0.7573333333333333,
      "calibration": 89.60646900269542,
      "cc": 51.46666666666666,
      "cc_by_type": {
       "noul": 51.46666666666666
      },
      "cc_tiered": 50.34494075753239,
      "cc_tiered_open": 53.325051759834366,
      "cc_tiered_sealed": 47.36482975523042,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 51.46666666666666
     },
     "score": {
      "accuracy": 0.6746666666666666,
      "calibration": 89.60530939476061,
      "cc": 55.4684210526316,
      "cc_by_type": {
       "score": 55.4684210526316
      },
      "cc_tiered": 56.029687617044516,
      "cc_tiered_open": 56.77505564115734,
      "cc_tiered_sealed": 55.28431959293169,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 55.4684210526316
     }
    },
    "raw_on_A": {
     "calibration": 86.48256493925484,
     "capability": 76.2896760781785,
     "intelligence": 66.09678721710216
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.52058715426469,
    "Emphasis on Accuracy 60:20:20": 62.78846844540931,
    "Emphasis on Cost 20:20:60": 57.98714486967113,
    "Emphasis on Speed 20:60:20": 71.13848048179798,
    "Intelligence only": 61.72139577013068
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Respan (lihe07)",
   "axes": {
    "calibration": 77.78014902808363,
    "cost": 53.528574234393844,
    "intelligence": 60.897967232257315,
    "speed": 85.7109495365315
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12956210701639312,
      "ece_score": 74.08757859672137,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 74.08757859672137,
      "score_before_time_limit": 74.08757859672137,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.1045576407506702,
      "brier": 0.12399376965004914,
      "ece": 0.09308935506900895,
      "n": 373,
      "score": 81.38212898619821,
      "score_before_time_limit": 81.38212898619821,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8655913978494624,
      "interval90_width": 0.22782258064516128,
      "n": 372,
      "nrps": 0.09086450416692268,
      "score": 77.87073950133129,
      "score_before_time_limit": 77.87073950133129,
      "time_limit_failed_share": 0,
      "top_ece": 0.17586035290322588
     }
    },
    "score": 77.78014902808363
   },
   "capability": 69.33905813017047,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 776,560 input + 102,328 output tokens over 1,485 decisions = 591.843771044 tokens/decision. GPU-time cost USD 0.0354036081682393/1,000; old list-price cost USD 0.0677365656565656/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0354036081682393/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0670592
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03540360816823925
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Respan Span-02",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-14",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned rented Lium GPU pod, 1x NVIDIA RTX 5090 32 GB (Romania), NOT the 1x H100 80 GB on which most of this draw's GPU rows were timed: the author's engine runs only on Blackwell (sm_120a), so this row's Speed axis rests on different hardware. The author's own Rust/CUDA dllm-server, built unchanged from their Dockerfile at the pinned revision (weights model.safetensors sha256 5105d39d..., tokenizer cc8d3a0c..., public MTP draft google/gemma-4-26B-A4B-it-assistant pre-staged), started by their own launch.sh with their documented Decision Index settings (container environment HOST=127.0.0.1, PORT=8000; weights mounted read-only at /model, the pre-staged draft at /data) in a container with --network none. Isolation timeline: the container's cut was proven by docker inspect and a failing outbound probe at 05:45:26Z (pod clock) before the sealed input file was copied to the pod host at 05:45:48Z (evaluator clock); the measuring container was then re-created with the same --network none setting and re-proven at 05:45:50Z before the first sealed item was sent at 05:45:55Z. The pod host itself kept its network for the evaluator's ssh session, as for every self-hosted row; no request left the container; one disclosed invocation change: launch.sh is stored without the executable bit in the repository (mode 100644), so the unchanged file was run through the interpreter its own shebang names (/bin/bash) instead of being exec'd directly. Driven by the unchanged run_v15 runner and TypeSafe adapter against /v1/systemone, sequentially, after a 4-item synthetic smoke gate. The standard self-hosted x2 + 0.15 s own-server adjustment (an assumption, not measured) applies on top of the raw p50/p95. 15 of 1,500 requests returned HTTP 400 because prompt + 256-token cap exceeds the server's 16,384-token context: exactly the 15 task ids pre-registered from an offline count with the author's own tokenizer (the same 15 that Attest-31B, Standard One 3B SH, Uniphore-4B-Decider and PurpleMIST-Mini failed on); they are scored as wrong in the full denominator. Measured 14 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b); pod removed afterwards. Regular-queue request GitHub issue #215; pre-registered spending and draw use.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 66.11779627322474,
    "I_sealed": 55.67813819128989,
    "base": 60.897967232257315,
    "excess": 0.6001313115791405,
    "gap": 10.439658081934851,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 76.60405175662127,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396664,
       "judge": 72.31377719182598,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 66.76949620427882,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 65.21739130434783,
       "judge": 70,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 54.979840858774125,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.446351153909085,
       "hard": 58.11316922436536,
       "judge": 51.62244140423914,
       "standard": 50.01602816182664
      }
     },
     "sealed|choice": {
      "cc": 74.26722280853451,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 68.77529779114143,
       "judge": 69.14789422135163,
       "standard": 81.01666315123391
      }
     },
     "sealed|noul": {
      "cc": 47.224888338629974,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 72.41379310344827,
       "hard": 34.54545454545455,
       "judge": 38.46153846153846,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 45.542303426705175,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 72.21811261531369,
       "hard": 38.81326566911587,
       "judge": 39.9318705079552,
       "standard": 54.078123725704486
      }
     }
    }
   },
   "jevbench_score": 67.0843877356023,
   "key": "respan-span-02",
   "last_measured_on": "2026-10-14",
   "licence": "Apache-2.0 per model card; weights are a Gemma 4 derivative under the Gemma 4 terms; NOTICE names its vLLM / SGLang / FlashInfer ports (Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.18",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "lihe07/span-02@3f0fbdb9c03950f335e80845f51be39985d63fc0",
   "noul_decisive": {
    "acc_among_decisive": 0.8682634730538922,
    "decisive": 334,
    "decisive_rate": 0.8954423592493298,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.78014902808363,
     "cost": 45.07530479505206,
     "intelligence": 60.897967232257315,
     "speed": 85.32322293910049
    },
    "board_release": "v1.6.43",
    "capability": 69.33905813017047,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      49.72091406052094,
      52.62443218967691
     ],
     "B": [
      48.771816312839555,
      52.66469779100122
     ],
     "C": [
      42.19743333671206,
      52.62443218967691
     ]
    },
    "jevbench_score": 51.45147163621532,
    "measured_in": "v1.6.18",
    "method": "v1.6",
    "open_board_rank": 40,
    "rank": 64,
    "ranks": {
     "A": 64,
     "B": 63,
     "C": 50,
     "capability": 83
    },
    "scores": {
     "A": 51.45147163621532,
     "B": 51.04738654109878,
     "C": 51.45147163621532
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20396152500179596,
     "p50_s_raw": 0.02698076250089798,
     "p95_s_adjusted": 1.4392275229169167,
     "p95_s_raw": 0.6446137614584584
    },
    "views": {
     "Balanced 33:33:33": 48.44678117072277,
     "Emphasis on Accuracy 60:20:20": 48.85975361687912,
     "Emphasis on Cost 20:20:60": 42.911545208581245,
     "Emphasis on Speed 20:60:20": 55.08693650119592,
     "Intelligence only": 49.49258431639605
    }
   },
   "rank": 100,
   "ranked": true,
   "ranks": {
    "A": 100,
    "B": 89,
    "C": 70,
    "capability": 105
   },
   "repo": "https://huggingface.co/lihe07/span-02",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.0843877356023,
    "B": 65.74855268918654,
    "C": 67.0843877356023
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21189434898260515,
    "p50_s_raw": 0.030947174491302576,
    "p95_s_adjusted": 1.2670264223954288,
    "p95_s_raw": 0.5585132111977145
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 63.655277626666674,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": -9.034167119999982
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 63.655277626666674
      },
      "calibration_gen": {
       "accuracy": 0.7128712871287128,
       "cc": 52.18918897588468,
       "cc_by_type": {
        "choice": 65.00530222693531,
        "noul": 28.000000000000004,
        "score": 63.56226470071873
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 52.18918897588468
      },
      "composite_gen": {
       "accuracy": 0.801980198019802,
       "cc": 62.123856121247435,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 33.33333333333333,
        "score": 62.79433259138462
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 62.123856121247435
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.42221393466112,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 99.26664180398333
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.42221393466112
      },
      "jaggedness": {
       "accuracy": 0.7388888888888889,
       "cc": 56.09877202708895,
       "cc_by_type": {
        "choice": 72.56977863330125,
        "noul": 45.945945945945944,
        "score": 49.78059150201963
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 56.09877202708895
      },
      "llm_router": {
       "accuracy": 0.8083067092651757,
       "cc": 69.63450226201023,
       "cc_by_type": {
        "choice": 74.64119045522585,
        "noul": 79.38144329896907,
        "score": 54.88087303183574
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 69.63450226201023
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7037037037037037,
       "cc": 36.21711239230192,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 50,
        "score": -16.606394781857148
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 36.21711239230192
      },
      "multilingual": {
       "accuracy": 0.7490909090909091,
       "cc": 57.90137613545051,
       "cc_by_type": {
        "choice": 72.61167264427779,
        "noul": 52.77777777777777,
        "score": 48.314677984295976
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 57.90137613545051
      },
      "negation_pair_gen": {
       "accuracy": 0.7674418604651163,
       "cc": 49.005011809555555,
       "cc_by_type": {
        "choice": 100,
        "noul": 26.66666666666666,
        "score": 20.348368762000003
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 49.005011809555555
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 83.85301706386177,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100,
        "score": 78.58607821861231
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 83.85301706386177
      },
      "usecase": {
       "accuracy": 0.6909090909090909,
       "cc": 53.00694751114175,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 50,
        "score": 35.981626847150736
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 53.00694751114175
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 29.139895762588512,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": -93.53269366461541
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 29.139895762588512
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 62.66211428368663,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 8.723670039999964
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 62.66211428368663
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 79.89665532626661,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100,
        "score": 79.51297482835739
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 79.89665532626661
      },
      "el": {
       "accuracy": 0.8,
       "cc": 72.17153475017349,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 33.33333333333333,
        "score": 99.94104745349999
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 72.17153475017349
      },
      "en": {
       "accuracy": 0.7636837532580365,
       "cc": 60.08592135150014,
       "cc_by_type": {
        "choice": 76.05594269047239,
        "noul": 52.517985611510795,
        "score": 51.68383575251725
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 60.08592135150014
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 50.27799962554451,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": -29.590144679647068
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 50.27799962554451
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 60.1156680291439,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 79.800556
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 60.1156680291439
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 84.44080642965491,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 74.49888987720001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.44080642965491
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 70.13113506375,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 75,
        "score": 35.39340519125
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 70.13113506375
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 52.55651539253981,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": -14.271217657142854
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 52.55651539253981
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 36.79395242405521,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 19.999999999999996,
        "score": 18.975979591466675
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 36.79395242405521
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 41.17323553835661,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 67.96415105951428
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 41.17323553835661
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.97295969931889,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.91887909795668
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.97295969931889
      },
      "nl": {
       "accuracy": 0.6,
       "cc": -6.864116600293109,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": -61.28499049352003
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 61.375750592128206,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 51.74468723977861
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 61.375750592128206
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 57.20084197883275,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 60.00000000000001,
        "score": 64.6372931727976
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 57.20084197883275
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 52.38715408758865,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 48.22529204999999
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 52.38715408758865
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 79.83776961083333,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 39.513308832499995
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 79.83776961083333
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 49.19090294308064,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 12.08883786149999
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 49.19090294308064
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 35.897047916258,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": -0.21846077100000638
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 35.897047916258
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8133333333333334,
       "cc": 75.88932049981823,
       "cc_by_type": {
        "choice": 75.88932049981823
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 75.88932049981823
      },
      "noul": {
       "accuracy": 0.7733333333333333,
       "cc": 54.666666666666664,
       "cc_by_type": {
        "noul": 54.666666666666664
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 54.666666666666664
      },
      "score": {
       "accuracy": 0.656,
       "cc": 49.01515268537382,
       "cc_by_type": {
        "score": 49.01515268537382
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 49.01515268537382
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": -14.728727007647068,
       "cc_by_type": {
        "choice": 52,
        "noul": -50,
        "score": -46.186181022941206
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 66.24337792218667,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 17.3636741392308
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 66.24337792218667
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 10.741343942373973,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": -18.995480368000027
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 10.741343942373973
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 3.962179066447362,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -51.27135753750001
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 3.962179066447362
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 22.902611299513293,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 31.026674478250005
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 22.902611299513293
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": -27.572823760356403,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": -190.89457191000002
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 21.835206443997695,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 9.203098323589732
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 21.835206443997695
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 24.485053083191342,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": -50,
        "score": 44.36108259452174
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 24.485053083191342
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 30.17254713148148,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 33.01764139444444
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 30.17254713148148
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 37.54539829080733,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 10.209356899545451
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 37.54539829080733
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 63.9457139474489,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 100,
        "score": 3.46504881909091
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 63.9457139474489
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 59.95658075553763,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100,
        "score": 76.64393581499999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 59.95658075553763
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 68.43503734801719,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 60.00000000000001,
        "score": 56.821426823322206
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.43503734801719
      },
      "model_routing": {
       "accuracy": 0.8433179723502304,
       "cc": 73.99832376743008,
       "cc_by_type": {
        "choice": 83.63636363636363,
        "noul": 96.96969696969697,
        "score": 41.388910696229665
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 73.99832376743008
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 77.61452788840782,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100,
        "score": 55.8762132237647
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 77.61452788840782
      },
      "recruiting": {
       "accuracy": 0.9444444444444444,
       "cc": 89.49197567076924,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 68.4759270123077
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 89.49197567076924
      },
      "risk_assessment": {
       "accuracy": 0.8333333333333334,
       "cc": 78.19399701137255,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 34.58199103411764
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 78.19399701137255
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 55.866288543441065,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 66.66666666666667,
        "score": 29.917706210033312
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 55.866288543441065
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.38080365366807,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.1424109610042
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.38080365366807
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.99882158581765,
      80.8791664084684
     ],
     "I": [
      55.16067369419073,
      65.320371533677
     ],
     "capability": [
      64.92778486069663,
      72.55981455085507
     ],
     "composite": null,
     "seed": 64
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.78014902808363,
     "capability": 69.33905813017047,
     "intelligence": 60.897967232257315
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8133333333333334,
      "calibration": 74.08757859672137,
      "cc": 75.88932049981823,
      "cc_by_type": {
       "choice": 75.88932049981823
      },
      "cc_tiered": 75.43563728257789,
      "cc_tiered_open": 76.60405175662127,
      "cc_tiered_sealed": 74.26722280853451,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.88932049981823
     },
     "noul": {
      "accuracy": 0.7733333333333333,
      "calibration": 81.38212898619821,
      "cc": 54.666666666666664,
      "cc_by_type": {
       "noul": 54.666666666666664
      },
      "cc_tiered": 56.997192271454395,
      "cc_tiered_open": 66.76949620427882,
      "cc_tiered_sealed": 47.224888338629974,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 54.666666666666664
     },
     "score": {
      "accuracy": 0.656,
      "calibration": 77.87073950133129,
      "cc": 49.01515268537382,
      "cc_by_type": {
       "score": 49.01515268537382
      },
      "cc_tiered": 50.26107214273965,
      "cc_tiered_open": 54.979840858774125,
      "cc_tiered_sealed": 45.542303426705175,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 49.01515268537382
     }
    },
    "raw_on_A": {
     "calibration": 77.50734571344526,
     "capability": 69.8865787495103,
     "intelligence": 62.265811785575366
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.1441740581115,
    "Emphasis on Accuracy 60:20:20": 62.8050284586991,
    "Emphasis on Cost 20:20:60": 59.429809207764656,
    "Emphasis on Speed 20:60:20": 71.32272579274326,
    "Intelligence only": 60.89796723225731
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "roudone (Hugging Face)",
   "axes": {
    "calibration": 86.57628518330837,
    "cost": 80.74092951874186,
    "intelligence": 27.068221597395038,
    "speed": 91.8816744041786
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0557088695588659,
      "ece_score": 88.85822608822681,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 88.85822608822681,
      "score_before_time_limit": 88.85822608822681,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.46380697050938335,
      "brier": 0.1673591152201377,
      "ece": 0.0885748488618805,
      "n": 373,
      "score": 82.2850302276239,
      "score_before_time_limit": 82.2850302276239,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.648969534050179,
      "n": 372,
      "nrps": 0.12178463171151906,
      "score": 88.58559923407441,
      "score_before_time_limit": 88.58559923407441,
      "time_limit_failed_share": 0,
      "top_ece": 0.05325169180349637
     }
    },
    "score": 86.57628518330837
   },
   "capability": 56.822253390351705,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 662,682 input + 0 output tokens over 1,485 decisions = 446.250505051 tokens/decision. GPU-time cost USD 0.00438499412707729/1,000; old list-price cost USD 0.0133875151515152/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00438499412707729/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.013253640000000002
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004384994127077292
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Decider-4B (roudone, NeoHorse 4B pointer model, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA A100-SXM4 80 GB (FIN-02, Finland, EU, pay-as-you-go non-spot, USD 2.168/h), shared by the two NeoHorse-family rows as two separate back-to-back uses (sealed input deleted from the node between them). The author's own server.py as shipped on stock PyPI vLLM 0.30.0 / torch 2.13.0 in a pre-built image, container with --network none (cut proven from inside before the sealed input), Hub offline, every credential unset, read-only bundle, all weight shards sha256-verified; driven by the unchanged pinned evaluator driver, one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 21 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 32.80120541792978,
    "I_sealed": 21.335237776860296,
    "base": 27.068221597395038,
    "excess": 1.6264408707137719,
    "gap": 11.465967641069483,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 55.18457786363322,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 45.265806857502355,
       "judge": 56.49307844429796,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 8.88198757763975,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 4.347826086956519,
       "judge": 0,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 34.33705081251637,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.81714109054795,
       "hard": 33.27535182513572,
       "judge": 32.334594052853205,
       "standard": 25.22408878775657
      }
     },
     "sealed|choice": {
      "cc": 43.02178780974991,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 58.06861499364677,
       "hard": 39.11183069272579,
       "judge": 44.46620959843293,
       "standard": 41.15165576882515
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 17.24137931034482,
       "hard": -19.999999999999996,
       "judge": -29.230769230769226,
       "standard": 4.477611940298498
      }
     },
     "sealed|score": {
      "cc": 20.98392552083098,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 32.75559389996422,
       "hard": 14.780780553889915,
       "judge": 16.42954167036802,
       "standard": 34.33595704084092
      }
     }
    }
   },
   "jevbench_score": 16.33572913211934,
   "key": "roudone-decider-4b",
   "last_measured_on": "2026-10-21",
   "licence": "Apache-2.0 (HF metadata and LICENSE), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.36",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "roudone/Decider-4B@a9a6a7924f9a6300c43b873df56414c087666083",
   "noul_decisive": {
    "acc_among_decisive": 0.895,
    "decisive": 200,
    "decisive_rate": 0.5361930294906166,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.57628518330837,
     "cost": 66.19900073922123,
     "intelligence": 27.068221597395038,
     "speed": 92.04108849933448
    },
    "board_release": "v1.6.43",
    "capability": 56.822253390351705,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 75,
    "capability_rank": 101,
    "composite_ci95": {
     "A": [
      9.819801762710863,
      22.011766792700506
     ],
     "B": [
      7.979052529165098,
      18.848733187935295
     ],
     "C": [
      6.8193067796603195,
      15.285949161597577
     ]
    },
    "jevbench_score": 15.743016217593315,
    "measured_in": "v1.6.36",
    "method": "v1.6",
    "open_board_rank": 119,
    "rank": 157,
    "ranks": {
     "A": 157,
     "B": 159,
     "C": 157,
     "capability": 155
    },
    "scores": {
     "A": 15.743016217593315,
     "B": 13.153182490278192,
     "C": 10.932650151106468
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2460378610002408,
     "p50_s_raw": 0.048018930500120405,
     "p95_s_adjusted": 0.25403245620000237,
     "p95_s_raw": 0.05201622810000117
    },
    "views": {
     "Balanced 33:33:33": 13.974968246133828,
     "Emphasis on Accuracy 60:20:20": 10.711684715357173,
     "Emphasis on Cost 20:20:60": 15.735373529584638,
     "Emphasis on Speed 20:60:20": 17.31228143677107,
     "Intelligence only": 7.933031176045365
    }
   },
   "rank": 199,
   "ranked": true,
   "ranks": {
    "A": 199,
    "B": 203,
    "C": 199,
    "capability": 187
   },
   "repo": "https://huggingface.co/roudone/Decider-4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.33572913211934,
    "B": 13.480095130715775,
    "C": 11.344256341749542
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24683192200009216,
    "p50_s_raw": 0.04841596100004608,
    "p95_s_adjusted": 0.262682556700247,
    "p95_s_raw": 0.0563412783501235
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Author server vllm-fast-text/server.py at a9a6a7924f9a6300c43b873df56414c087666083 (HEAD later moved to ee44f389 = README.md only; every .py and weight oid identical), started directly as shipped (independent static review PASS WITH FINDINGS, 0 blockers; no subprocess/network in the server import path; run_evaluation.py, evaluate.py and provenance/* never started). Author defaults kept: position pooling OFF, no prefix or head cache. BASE: the repository's backbone/config.json modification notice and its submission text state that the weights were fine-tuned from Qwen/Qwen3.5-4B (tensor values unchanged by repackaging), while the card's base_model field names TokenRhythm/NeoHorse-1-4B and other metadata names a different NeoHorse name or a local path; the base string Qwen/Qwen3.5-4B is used because it is the upstream model the repository itself states and the only one the frozen pricing reference can resolve. The authors' lineage claims are inconsistent and unverified. FIRST MEASUREMENT (free-queue submission E94B190D; not the unrelated Mapika/decider-4b row). CONTEXT CAP: the author server refuses requests whose state exceeds 8,192 tokens or whose state+question row exceeds 12,288 tokens (HTTP 422); on this draw that is exactly the 15 items of 76k-81k tokens, pre-registered by id and position; they count as wrong, like d3-turbo, elf-1-12b and seb-9b on this draw. SAME FAMILY: Goddragon/Arbiter-4B and roudone/Decider-4B are two different checkpoints submitted through the free queue that both serve as 'NeoHorse-Jev-4B' (TYPESAFE_MODEL alias set for the frozen adapter). PROVENANCE, author statements (unverified): base lineage TokenRhythm/NeoHorse-1-4B / Qwen3.5-4B; training on the typed-decisions train split plus other collected decision datasets (README). No rule against public items was found in the reviewed code. HARDWARE: A100 80 GB (the RTX PRO 6000 was out of stock); latency reflects that card. Card metadata names the base inconsistently (NeoHorse-1-4B / NeoHorse-Jev-4B / a local path).",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 34.012771811750206,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 50,
        "score": -6.295017898082733
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 34.012771811750206
      },
      "calibration_gen": {
       "accuracy": 0.37623762376237624,
       "cc": -4.198761387173277,
       "cc_by_type": {
        "choice": 14.103923647932142,
        "noul": -28.000000000000004,
        "score": 1.2997921905480325
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 6.945410137580818,
       "cc_by_type": {
        "choice": 54.47154471544716,
        "noul": -48.14814814814815,
        "score": 14.512833845443451
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 6.945410137580818
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 73.90354996650584,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 61.7106498995175
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 73.90354996650584
      },
      "jaggedness": {
       "accuracy": 0.32222222222222224,
       "cc": -10.471287065762853,
       "cc_by_type": {
        "choice": 16.265640038498567,
        "noul": -56.75675675675676,
        "score": 9.07725552096963
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.7380191693290735,
       "cc": 57.31067969597308,
       "cc_by_type": {
        "choice": 65.13163687593556,
        "noul": 60.82474226804124,
        "score": 45.975659943942446
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 57.31067969597308
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -8.559150937831872,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -50,
        "score": 10.920485330834284
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5418181818181819,
       "cc": 21.37159547449154,
       "cc_by_type": {
        "choice": 48.15780893381155,
        "noul": -22.22222222222222,
        "score": 38.17919971188529
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 21.37159547449154
      },
      "negation_pair_gen": {
       "accuracy": 0.313953488372093,
       "cc": -23.76484736800194,
       "cc_by_type": {
        "choice": 26.666666666666668,
        "noul": -86.66666666666667,
        "score": -11.294542104005822
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.7916666666666666,
       "cc": 73.36884154460719,
       "cc_by_type": {
        "choice": 73.36884154460719
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 73.36884154460719
      },
      "perturb_pair": {
       "accuracy": 0.6875,
       "cc": 65.4362645610416,
       "cc_by_type": {
        "choice": 48.94894894894895,
        "noul": 100,
        "score": 47.35984473417586
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 65.4362645610416
      },
      "usecase": {
       "accuracy": 0.45454545454545453,
       "cc": 16.25684731361378,
       "cc_by_type": {
        "choice": 41.17647058823531,
        "noul": -16.666666666666664,
        "score": 24.260738019272686
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 16.25684731361378
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 4.470290525637845,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0,
        "score": -10.398652232610273
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 4.470290525637845
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.38461538461538464,
       "cc": -1.9166212453026752,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -33.333333333333336,
        "score": -10.204548835754412
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.71875,
       "cc": 54.14494959379758,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 50,
        "score": 44.29325586103874
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 54.14494959379758
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 32.05475206640077,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -33.333333333333336,
        "score": 96.53669567778707
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 32.05475206640077
      },
      "en": {
       "accuracy": 0.5204170286707211,
       "cc": 20.86007070055345,
       "cc_by_type": {
        "choice": 44.64309686237655,
        "noul": -4.316546762589923,
        "score": 22.25366200187372
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 20.86007070055345
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 62.411720719403775,
       "cc_by_type": {
        "choice": 60.848287112561174,
        "noul": 100,
        "score": 26.38687504565015
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 62.411720719403775
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -6.872585540695051,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -33.333333333333336,
        "score": 43.863117694854736
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 44.61722545908915,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 55.028146965502714
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 44.61722545908915
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": 3.883462078310893,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -50,
        "score": 11.929715843871225
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 3.883462078310893
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 1,
       "cc": 87.33703584543296,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 62.01110753629888
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 87.33703584543296
      },
      "ja": {
       "accuracy": 0.52,
       "cc": 25.251135116934716,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": 19.999999999999996,
        "score": 41.535772308707244
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 25.251135116934716
      },
      "ko": {
       "accuracy": 0.2,
       "cc": -24.507053960213273,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": 15.367727008249066
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.7857142857142857,
       "cc": 60.47832132317126,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 64.7682973028471
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 60.47832132317126
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 39.71882606567587,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 78.4638375043869
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 39.71882606567587
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.48717948717948717,
       "cc": 13.214770734569521,
       "cc_by_type": {
        "choice": 54.29538721963606,
        "noul": -40,
        "score": 25.348924984072507
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 13.214770734569521
      },
      "pt": {
       "accuracy": 0.6071428571428571,
       "cc": 23.55856931728997,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": -19.999999999999996,
        "score": 43.71047518816926
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 23.55856931728997
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 3.5609835000536996,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 47.278695181012154
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 3.5609835000536996
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": 17.000425566473734,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": -5.520462431013584
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 17.000425566473734
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 62.605922524967504,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 52.33389660716057
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 62.605922524967504
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 9.519007465961737,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -12.68591545522213
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.519007465961737
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.58,
       "cc": 45.750971124590976,
       "cc_by_type": {
        "choice": 45.750971124590976
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 45.750971124590976
      },
      "noul": {
       "accuracy": 0.47733333333333333,
       "cc": -4.533333333333333,
       "cc_by_type": {
        "noul": -4.533333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.488,
       "cc": 25.22129282528254,
       "cc_by_type": {
        "score": 25.22129282528254
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 25.22129282528254
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -24.995524754711237,
       "cc_by_type": {
        "choice": 20,
        "noul": -50,
        "score": -44.98657426413371
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": 4.0585434016706685,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -33.333333333333336,
        "score": 1.4095846563577652
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 4.0585434016706685
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 40.22811272441533,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50,
        "score": 35.72498857975006
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 40.22811272441533
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -4.523927283057579,
       "cc_by_type": {
        "choice": -16.959064327485383,
        "noul": -33.333333333333336,
        "score": 36.720615811645985
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": -0.9475624314756766,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100,
        "score": 55.12832719832659
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -23.57661018081681,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100,
        "score": 17.320483922958374
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.43333333333333335,
       "cc": 8.757336831743325,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 27.11234662968376
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 8.757336831743325
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -18.679188344642103,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100,
        "score": 17.133166673390765
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 47.77736546402728,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 60.00000000000001,
        "score": 66.66542972541518
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 47.77736546402728
      },
      "knowledge_graphs": {
       "accuracy": 0.18181818181818182,
       "cc": -10.357701631714406,
       "cc_by_type": {
        "choice": -15.631691648822272,
        "noul": -33.333333333333336,
        "score": 17.89192008701239
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 24.545290011881367,
       "cc_by_type": {
        "choice": 65.11627906976744,
        "noul": -11.111111111111116,
        "score": 19.63070207698778
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 24.545290011881367
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -15.709171836735104,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": 17.388613522052754
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 27.026718143016733,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": 19.999999999999996,
        "score": 18.66172832540338
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 27.026718143016733
      },
      "model_routing": {
       "accuracy": 0.7603686635944701,
       "cc": 60.19956873120693,
       "cc_by_type": {
        "choice": 71.94805194805195,
        "noul": 72.72727272727273,
        "score": 35.92338151829608
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 60.19956873120693
      },
      "moderation_trust_safety": {
       "accuracy": 0.4444444444444444,
       "cc": -7.464831114845926,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100,
        "score": 35.187080551815384
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.5555555555555556,
       "cc": 38.52552177678152,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 15.66941584102237
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 38.52552177678152
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 8.725645628783953,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50,
        "score": 5.73358713265727
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 8.725645628783953
      },
      "scientific_discovery": {
       "accuracy": 0.30434782608695654,
       "cc": -15.951830414956177,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -66.66666666666667,
        "score": 20.26045078411697
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 85.08908549209596,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100,
        "score": 76.61956964354766
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 85.08908549209596
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.98866101809301,
      87.30669227235894
     ],
     "I": [
      22.51368521464031,
      31.065507713750073
     ],
     "capability": [
      52.217894521611484,
      58.33910547313941
     ],
     "composite": null,
     "seed": 65
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.57628518330837,
     "capability": 56.822253390351705,
     "intelligence": 27.068221597395038
    },
    "per_type": {
     "choice": {
      "accuracy": 0.58,
      "calibration": 88.85822608822681,
      "cc": 45.750971124590976,
      "cc_by_type": {
       "choice": 45.750971124590976
      },
      "cc_tiered": 49.103182836691566,
      "cc_tiered_open": 55.18457786363322,
      "cc_tiered_sealed": 43.02178780974991,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 45.750971124590976
     },
     "noul": {
      "accuracy": 0.47733333333333333,
      "calibration": 82.2850302276239,
      "cc": -4.533333333333333,
      "cc_by_type": {
       "noul": -4.533333333333333
      },
      "cc_tiered": 4.440993788819875,
      "cc_tiered_open": 8.88198757763975,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.488,
      "calibration": 88.58559923407441,
      "cc": 25.22129282528254,
      "cc_by_type": {
       "score": 25.22129282528254
      },
      "cc_tiered": 27.660488166673673,
      "cc_tiered_open": 34.33705081251637,
      "cc_tiered_sealed": 20.98392552083098,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 25.22129282528254
     }
    },
    "raw_on_A": {
     "calibration": 83.84888033371581,
     "capability": 56.08107835039779,
     "intelligence": 28.31327636707977
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.602045710610248,
    "Emphasis on Accuracy 60:20:20": 10.927502183783087,
    "Emphasis on Cost 20:20:60": 17.243154509363173,
    "Emphasis on Speed 20:60:20": 17.874885413898586,
    "Intelligence only": 7.933031176045365
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Shanghua Gao (@GaoShanghua)",
   "axes": {
    "calibration": 85.34979344070516,
    "cost": 58.679516754522645,
    "intelligence": 61.564156247760415,
    "speed": 84.6270713604755
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08475596558784464,
      "ece_score": 83.04880688243107,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 83.04880688243107,
      "score_before_time_limit": 83.04880688243107,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.17307692307692313,
      "brier": 0.10655681075571934,
      "ece": 0.06711648937487673,
      "n": 364,
      "score": 86.57670212502465,
      "score_before_time_limit": 86.57670212502465,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9402173913043478,
      "interval90_width": 0.263586956521739,
      "n": 368,
      "nrps": 0.07024138314041055,
      "score": 86.42387131465975,
      "score_before_time_limit": 86.42387131465975,
      "time_limit_failed_share": 0,
      "top_ece": 0.10064059528319728
     }
    },
    "score": 85.34979344070516
   },
   "capability": 73.45697484423279,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 694,381 input + 1,471 output tokens over 1,500 decisions = 463.901333333 tokens/decision. GPU-time cost USD 0.0238423709982921/1,000; old list-price cost USD 0.19736868/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0238423709982921/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02384237099829211
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "RSI-Jev v6.1-VL 27B",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium H100 80GB pod (36 min, USD 1.95/h), sealed under the approved pre-registration: the author's own serve.cli inside a loopback-only network namespace (unshare -n; DNS and outbound TCP both proven dead from inside the run namespace), started with env -i and OPENAI_API_KEY empty, plus a pod-level iptables egress cut installed and verified before the sealed input was uploaded. HF_HUB_OFFLINE/TRANSFORMERS_OFFLINE set only to block the download fallbacks the r75 review found; no source line was edited. Server pointed at the LOCAL PINNED SNAPSHOT DIRECTORY, not a repo id, so no revision could resolve to main. Unchanged defaults, --effort auto, 32,768-token cap, no truncation. 54,047 MiB of 81,559 MiB used.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 68.48613704048307,
    "I_sealed": 54.64217545503776,
    "base": 61.564156247760415,
    "excess": 3.7707170041056006,
    "gap": 13.843961585445314,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.8256733782429,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 73.57659641396666,
       "judge": 72.31377719182598,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 66.15458937198068,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 39.99999999999999,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 63.478148371225664,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.937072018553,
       "hard": 66.44443112298649,
       "judge": 61.06082661163181,
       "standard": 58.942103683431114
      }
     },
     "sealed|choice": {
      "cc": 70.33653729047333,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 88.73709458754823,
       "hard": 64.87853138397273,
       "judge": 71.69093399624576,
       "standard": 70.02067539627843
      }
     },
     "sealed|noul": {
      "cc": 41.66298613667034,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 58.974358974358964,
       "hard": 21.212121212121215,
       "judge": 50,
       "standard": 61.403508771929815
      }
     },
     "sealed|score": {
      "cc": 51.92700293796962,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 68.04656098024147,
       "hard": 33.730600016105896,
       "judge": 64.21257035928524,
       "standard": 61.83167862858772
      }
     }
    }
   },
   "jevbench_score": 70.4002508405647,
   "key": "rsi-jev-v6.1-vl-27b",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (model card); source Shanghua-Gao/RSI-Jev@fc982bbd MIT",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "shgao/rsi-jev-v6.1-vl-27b@d0dccc052ee844ab7481ddfacad1b7335ab394e0",
   "noul_decisive": {
    "acc_among_decisive": 0.9102990033222591,
    "decisive": 301,
    "decisive_rate": 0.8269230769230769,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.34979344070516,
     "cost": 31.141652902933714,
     "intelligence": 61.564156247760415,
     "speed": 84.71397705257328
    },
    "board_release": "v1.6.43",
    "capability": 73.45697484423279,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      21.062859904802277,
      21.88568498833731
     ],
     "B": [
      21.239170136076478,
      22.480828485144727
     ],
     "C": [
      18.819765630154432,
      21.88568498833731
     ]
    },
    "jevbench_score": 21.58850717988564,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 87,
    "rank": 120,
    "ranks": {
     "A": 120,
     "B": 116,
     "C": 102,
     "capability": 56
    },
    "scores": {
     "A": 21.58850717988564,
     "B": 22.011280090800952,
     "C": 21.58850717988564
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.39549588703666816,
     "p50_s_raw": 0.12274794351833407,
     "p95_s_adjusted": 0.8540048229158855,
     "p95_s_raw": 0.35200241145794275
    },
    "views": {
     "Balanced 33:33:33": 19.344794057306224,
     "Emphasis on Accuracy 60:20:20": 20.935791627644683,
     "Emphasis on Cost 20:20:60": 15.593981583505036,
     "Emphasis on Speed 20:60:20": 23.15453324264071,
     "Intelligence only": 23.882030176915077
    }
   },
   "rank": 72,
   "ranked": true,
   "ranks": {
    "A": 72,
    "B": 67,
    "C": 56,
    "capability": 73
   },
   "repo": "https://huggingface.co/shgao/rsi-jev-v6.1-vl-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.4002508405647,
    "B": 68.43577975762766,
    "C": 70.4002508405647
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4150659809878562,
    "p50_s_raw": 0.13253299049392808,
    "p95_s_adjusted": 0.830186609426164,
    "p95_s_raw": 0.34009330471308197
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9310344827586207,
       "cc": 86.4125424016998,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 92.5709605384327
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 86.4125424016998
      },
      "calibration_gen": {
       "accuracy": 0.7232142857142857,
       "cc": 58.19506668349169,
       "cc_by_type": {
        "choice": 67.3913043478261,
        "noul": 61.53846153846154,
        "score": 45.655434164187426
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 58.19506668349169
      },
      "composite_gen": {
       "accuracy": 0.8877551020408163,
       "cc": 82.46075900003056,
       "cc_by_type": {
        "choice": 96.29629629629629,
        "noul": 78.57142857142858,
        "score": 72.51455213236682
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 82.46075900003056
      },
      "fairness": {
       "accuracy": 0.9629629629629629,
       "cc": 94.1963028792578,
       "cc_by_type": {
        "choice": 88.23529411764707,
        "noul": 100,
        "score": 94.35361452012633
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 94.1963028792578
      },
      "jaggedness": {
       "accuracy": 0.5081081081081081,
       "cc": 13.570657148702269,
       "cc_by_type": {
        "choice": 38.86058360352014,
        "noul": -48.71794871794872,
        "score": 50.56933656053539
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 13.570657148702269
      },
      "llm_router": {
       "accuracy": 0.8598484848484849,
       "cc": 81.63698812259595,
       "cc_by_type": {
        "choice": 75.84765443567116,
        "noul": 86.30136986301369,
        "score": 82.76194006910303
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 81.63698812259595
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6774193548387096,
       "cc": 31.38414817065086,
       "cc_by_type": {
        "choice": 72.30769230769232,
        "noul": 0,
        "score": 21.844752204260253
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 31.38414817065086
      },
      "multilingual": {
       "accuracy": 0.7428571428571429,
       "cc": 58.6257780124704,
       "cc_by_type": {
        "choice": 70.44334975369458,
        "noul": 51.72413793103448,
        "score": 53.70984635268214
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 58.6257780124704
      },
      "negation_pair_gen": {
       "accuracy": 0.875,
       "cc": 63.20741226657981,
       "cc_by_type": {
        "choice": 90.47619047619048,
        "noul": 60.00000000000001,
        "score": 39.146046323548944
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 63.20741226657981
      },
      "option_order": {
       "accuracy": 0.9824561403508771,
       "cc": 97.78024417314094,
       "cc_by_type": {
        "choice": 97.78024417314094
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 97.78024417314094
      },
      "perturb_pair": {
       "accuracy": 0.9,
       "cc": 89.49429032954704,
       "cc_by_type": {
        "choice": 84.27260812581913,
        "noul": 100,
        "score": 84.21026286282202
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 89.49429032954704
      },
      "usecase": {
       "accuracy": 0.7046413502109705,
       "cc": 51.83295324947296,
       "cc_by_type": {
        "choice": 73.40528040084796,
        "noul": 36.8421052631579,
        "score": 45.251474084413026
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 51.83295324947296
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8181818181818182,
       "cc": 69.11348259490397,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 50,
        "score": 84.00711445137858
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 69.11348259490397
      },
      "cs": {
       "accuracy": 0.875,
       "cc": 87.28813559322035,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 87.28813559322035
      },
      "da": {
       "accuracy": 0.6666666666666666,
       "cc": 44.00477315138941,
       "cc_by_type": {
        "choice": 52.662721893491124,
        "noul": 19.999999999999996,
        "score": 59.351597560677114
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 44.00477315138941
      },
      "de": {
       "accuracy": 0.8444444444444444,
       "cc": 72.3799653014141,
       "cc_by_type": {
        "choice": 83.70549185274592,
        "noul": 77.77777777777777,
        "score": 55.656626273718615
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 72.3799653014141
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": -0.8800661948371129,
       "cc_by_type": {
        "choice": 75.90361445783132,
        "noul": -19.999999999999996,
        "score": -58.54381304234266
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.7497765862377123,
       "cc": 57.06152962458927,
       "cc_by_type": {
        "choice": 72.18171620863711,
        "noul": 42.85714285714286,
        "score": 56.14572980798784
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 57.06152962458927
      },
      "es": {
       "accuracy": 0.8928571428571429,
       "cc": 82.61244146186118,
       "cc_by_type": {
        "choice": 86.90909090909093,
        "noul": 100,
        "score": 60.9282334764926
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 82.61244146186118
      },
      "fi": {
       "accuracy": 0.5555555555555556,
       "cc": 34.97652582159624,
       "cc_by_type": {
        "choice": 36.61971830985915,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 34.97652582159624
      },
      "fr": {
       "accuracy": 0.6666666666666666,
       "cc": 41.96733456321797,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 0,
        "score": 51.28987392942821
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 41.96733456321797
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 52.85728930896761,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 60.00000000000001,
        "score": 35.83894867224444
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 52.85728930896761
      },
      "id": {
       "accuracy": 0.8333333333333334,
       "cc": 83.87096774193549,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 83.87096774193549
      },
      "it": {
       "accuracy": 1,
       "cc": 90.29663262626855,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 70.88989787880564
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 90.29663262626855
      },
      "ja": {
       "accuracy": 0.53125,
       "cc": 32.05622061795162,
       "cc_by_type": {
        "choice": 43.08943089430895,
        "noul": 14.28571428571428,
        "score": 38.79351667383163
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 32.05622061795162
      },
      "ko": {
       "accuracy": 0.8181818181818182,
       "cc": 85.45781546660389,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 56.373446399811655
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 85.45781546660389
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.9689563002903,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.9068689008709
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 99.9689563002903
      },
      "nl": {
       "accuracy": 0.6923076923076923,
       "cc": 40.991198714143415,
       "cc_by_type": {
        "choice": 74.93036211699165,
        "noul": 33.33333333333333,
        "score": 14.709900692105283
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 40.991198714143415
      },
      "no": {
       "accuracy": 0.7142857142857143,
       "cc": 67.21311475409837,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 67.21311475409837
      },
      "pl": {
       "accuracy": 0.7,
       "cc": 51.32398893140217,
       "cc_by_type": {
        "choice": 92.12942719720158,
        "noul": 19.999999999999996,
        "score": 41.84253959700492
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 51.32398893140217
      },
      "pt": {
       "accuracy": 0.8333333333333334,
       "cc": 83.10065347581066,
       "cc_by_type": {
        "choice": 63.81909547738693,
        "noul": 100,
        "score": 85.48286495004504
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 83.10065347581066
      },
      "sv": {
       "accuracy": 0.625,
       "cc": 53.29148096066934,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 50,
        "score": 98.76333177089691
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 53.29148096066934
      },
      "tr": {
       "accuracy": 0.7272727272727273,
       "cc": 23.85319314395998,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 33.33333333333333,
        "score": -35.457964427769184
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 23.85319314395998
      },
      "uk": {
       "accuracy": 0.5454545454545454,
       "cc": 4.1198605058986955,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100,
        "score": 77.5769728220439
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 4.1198605058986955
      },
      "zh": {
       "accuracy": 0.875,
       "cc": 73.56783818557031,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 33.33333333333333,
        "score": 87.37018122337759
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 73.56783818557031
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7826666666666666,
       "cc": 71.84315080324754,
       "cc_by_type": {
        "choice": 71.84315080324754
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 71.84315080324754
      },
      "noul": {
       "accuracy": 0.7306666666666667,
       "cc": 46.13333333333334,
       "cc_by_type": {
        "noul": 46.13333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 46.13333333333334
      },
      "score": {
       "accuracy": 0.6986666666666667,
       "cc": 54.531190100580204,
       "cc_by_type": {
        "score": 54.531190100580204
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 54.531190100580204
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7083333333333334,
       "cc": 36.966773340637815,
       "cc_by_type": {
        "choice": 70.34596375617794,
        "noul": -19.999999999999996,
        "score": 60.55435626573551
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 36.966773340637815
      },
      "code_linting": {
       "accuracy": 0.75,
       "cc": 68.95860502475776,
       "cc_by_type": {
        "choice": 70.54009819967267,
        "noul": 100,
        "score": 36.33571687460061
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 68.95860502475776
      },
      "customer_support": {
       "accuracy": 0.75,
       "cc": 47.499513085946944,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 75,
        "score": -0.24339622603013478
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 47.499513085946944
      },
      "demand_forecasting": {
       "accuracy": 0.3076923076923077,
       "cc": -17.782585557336656,
       "cc_by_type": {
        "choice": 15.662650602409636,
        "noul": -50,
        "score": -19.010407274419606
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5555555555555556,
       "cc": 28.57990326207394,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -19.999999999999996,
        "score": 43.006790531563425
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 28.57990326207394
      },
      "feature_extraction": {
       "accuracy": 0.8620689655172413,
       "cc": 64.36666498139438,
       "cc_by_type": {
        "choice": 82.90598290598291,
        "noul": 19.999999999999996,
        "score": 90.19401203820024
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 64.36666498139438
      },
      "financial_crime": {
       "accuracy": 0.7692307692307693,
       "cc": 69.24951206795548,
       "cc_by_type": {
        "choice": 86.95652173913045,
        "noul": 100,
        "score": 20.792014464735985
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 69.24951206795548
      },
      "gaming": {
       "accuracy": 0.5862068965517241,
       "cc": 25.37172392511331,
       "cc_by_type": {
        "choice": 52.475247524752476,
        "noul": -14.28571428571429,
        "score": 37.92563853630174
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 25.37172392511331
      },
      "insurance_claims": {
       "accuracy": 0.5714285714285714,
       "cc": 31.669815093397148,
       "cc_by_type": {
        "choice": 52.38095238095238,
        "noul": -19.999999999999996,
        "score": 62.62849289923906
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 31.669815093397148
      },
      "knowledge_graphs": {
       "accuracy": 0.6956521739130435,
       "cc": 42.704203038079555,
       "cc_by_type": {
        "choice": 72.51908396946564,
        "noul": 19.999999999999996,
        "score": 35.59352514477303
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 42.704203038079555
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 55.47104563667543,
       "cc_by_type": {
        "choice": 100,
        "noul": 25,
        "score": 41.413136910026275
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 55.47104563667543
      },
      "legal_compliance": {
       "accuracy": 0.6923076923076923,
       "cc": 8.840061259572431,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": -100,
        "score": 59.30706902461893
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 8.840061259572431
      },
      "llm_guardrails": {
       "accuracy": 0.9310344827586207,
       "cc": 89.90702297249,
       "cc_by_type": {
        "choice": 90.8256880733945,
        "noul": 100,
        "score": 78.89538084407552
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 89.90702297249
      },
      "model_routing": {
       "accuracy": 0.86,
       "cc": 76.7397155888396,
       "cc_by_type": {
        "choice": 85.52666845349772,
        "noul": 81.81818181818181,
        "score": 62.874296494839264
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 76.7397155888396
      },
      "moderation_trust_safety": {
       "accuracy": 0.6875,
       "cc": 52.07409272558595,
       "cc_by_type": {
        "choice": 52.63157894736842,
        "noul": 60.00000000000001,
        "score": 43.590699229389415
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 52.07409272558595
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 54.013036775459085,
       "cc_by_type": {
        "choice": 54.77386934673366,
        "noul": 66.66666666666667,
        "score": 40.59857431297692
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 54.013036775459085
      },
      "risk_assessment": {
       "accuracy": 0.6153846153846154,
       "cc": 46.84943347419826,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 33.33333333333333,
        "score": 44.95081614586523
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 46.84943347419826
      },
      "scientific_discovery": {
       "accuracy": 0.59375,
       "cc": 25.763079128747716,
       "cc_by_type": {
        "choice": 60.1593625498008,
        "noul": -14.28571428571429,
        "score": 31.41558912215664
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 25.763079128747716
      },
      "search_retrieval": {
       "accuracy": 0.9090909090909091,
       "cc": 86.99121578361938,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 60.97364735085814
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 86.99121578361938
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.64755807175129,
      87.47437139642248
     ],
     "I": [
      56.725930974421935,
      65.23484456387472
     ],
     "capability": [
      69.35974616373294,
      75.75238339213846
     ],
     "composite": null,
     "seed": 27
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.34979344070516,
     "capability": 73.45697484423279,
     "intelligence": 61.564156247760415
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7826666666666666,
      "calibration": 83.04880688243107,
      "cc": 71.84315080324754,
      "cc_by_type": {
       "choice": 71.84315080324754
      },
      "cc_tiered": 73.08110533435811,
      "cc_tiered_open": 75.8256733782429,
      "cc_tiered_sealed": 70.33653729047333,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 71.84315080324754
     },
     "noul": {
      "accuracy": 0.7306666666666667,
      "calibration": 86.57670212502465,
      "cc": 46.13333333333334,
      "cc_by_type": {
       "noul": 46.13333333333334
      },
      "cc_tiered": 53.90878775432551,
      "cc_tiered_open": 66.15458937198068,
      "cc_tiered_sealed": 41.66298613667034,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 46.13333333333334
     },
     "score": {
      "accuracy": 0.6986666666666667,
      "calibration": 86.42387131465975,
      "cc": 54.531190100580204,
      "cc_by_type": {
       "score": 54.531190100580204
      },
      "cc_tiered": 57.70257565459764,
      "cc_tiered_open": 63.478148371225664,
      "cc_tiered_sealed": 51.92700293796962,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.531190100580204
     }
    },
    "raw_on_A": {
     "calibration": 84.95320234939237,
     "capability": 73.13630404134082,
     "intelligence": 61.31940573328927
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.51665035369663,
    "Emphasis on Accuracy 60:20:20": 64.44301736262479,
    "Emphasis on Cost 20:20:60": 63.14332551836284,
    "Emphasis on Speed 20:60:20": 72.743579020478,
    "Intelligence only": 61.564156247760415
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "csabag",
   "axes": {
    "calibration": 73.68644793817732,
    "cost": null,
    "intelligence": 35.358086407070836,
    "speed": 89.07386048376334
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12927287978799915,
      "ece_score": 74.14542404240017,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 73.15681838850149,
      "score_before_time_limit": 74.14542404240017,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.2666666666666667,
      "brier": 0.18679415520437917,
      "ece": 0.15039467431582756,
      "n": 375,
      "score": 69.54815278943805,
      "score_before_time_limit": 69.9210651368345,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.8853333333333333,
      "interval90_width": 0.40755555555555545,
      "n": 375,
      "nrps": 0.12029838922886724,
      "score": 78.35437263659244,
      "score_before_time_limit": 78.98626273850044,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.14998817800056186
     }
    },
    "score": 73.68644793817732
   },
   "capability": 54.52226717262408,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 3948e3f1..., and the later re-freezes, which only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base google/gemma-4-E4B-it (the module RAISES 'base model lacks an explicit market-listing decision'); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens as reported by the author's server (input 2,095,834, output_tokens 0).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "RYOTIDE-Gemma (csabag, Gemma-4-E4B stock weights)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own server, next-token softmax over the option markers, T 1.924.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.9529909992295,
    "I_sealed": 28.763181814912173,
    "base": 35.358086407070836,
    "excess": 3.350282413961615,
    "gap": 13.189809184317326,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 57.132088899550936,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 54.70273670965713,
       "judge": 60.44825313117996,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 26.634230503795727,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": -9.999999999999998,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 42.092653594341826,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.14359632238972,
       "hard": 33.421649824569585,
       "judge": 49.75696822001671,
       "standard": 43.912717831350044
      }
     },
     "sealed|choice": {
      "cc": 50.73087956914682,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 69.50444726810673,
       "hard": 43.79553602405458,
       "judge": 55.04407443682666,
       "standard": 48.74499050833158
      }
     },
     "sealed|noul": {
      "cc": 12.45512490597411,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": 0,
       "judge": 4.615384615384621,
       "standard": 43.28358208955223
      }
     },
     "sealed|score": {
      "cc": 23.103540969615587,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 51.41157137929402,
       "hard": 9.075703914379096,
       "judge": 24.22274243295568,
       "standard": 35.32639768023921
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "ryotide-gemma",
   "last_measured_on": "2026-10-16",
   "licence": "per base model (stock weights; code licence not checked)",
   "listing": "listed",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "csabag/ryotide@6bbb615a170dc16b83ca62373aa7ada7bf3ea27e",
   "not_ranked_because": "unpriced: the frozen pricing path has no market-listing decision for the declared base google/gemma-4-E4B-it and raises rather than estimating; one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.8109090909090909,
    "decisive": 275,
    "decisive_rate": 0.7372654155495979,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 74.3509173059117,
     "cost": null,
     "intelligence": 36.06622980750019,
     "speed": 89.15460718023942
    },
    "board_release": "v1.6.43",
    "capability": 55.20857355670594,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3395472984761,
     "p50_s_raw": 0.09477364923804998,
     "p95_s_adjusted": 0.35779890120029445,
     "p95_s_raw": 0.10389945060014724
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://github.com/csabag/ryotide",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.34042168045416477,
    "p50_s_raw": 0.09521084022708237,
    "p95_s_adjusted": 0.3635772834531963,
    "p95_s_raw": 0.10678864172659815
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 78.53492076277824,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 68.93809562166804
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 78.53492076277824
      },
      "calibration_gen": {
       "accuracy": 0.37623762376237624,
       "cc": -3.447323116310777,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": -36,
        "score": -7.533909963990659
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.48514851485148514,
       "cc": 9.36975394625747,
       "cc_by_type": {
        "choice": 44.71544715447154,
        "noul": -25.92592592592593,
        "score": 9.319740610226802
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 9.36975394625747
      },
      "fairness": {
       "accuracy": 1,
       "cc": 96.9575143880138,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 90.87254316404142
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 96.9575143880138
      },
      "jaggedness": {
       "accuracy": 0.39444444444444443,
       "cc": 0.6570258968792307,
       "cc_by_type": {
        "choice": 32.14629451395574,
        "noul": -40.54054054054054,
        "score": 10.365323717222497
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0.6570258968792307
      },
      "llm_router": {
       "accuracy": 0.7476038338658147,
       "cc": 63.635389162713636,
       "cc_by_type": {
        "choice": 55.622083296645236,
        "noul": 75.25773195876289,
        "score": 60.02635223273278
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 63.635389162713636
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": 1.9570792379214061,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -25,
        "score": 17.46917585809411
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 1.9570792379214061
      },
      "multilingual": {
       "accuracy": 0.6654545454545454,
       "cc": 40.158140045525016,
       "cc_by_type": {
        "choice": 68.6990544506032,
        "noul": 13.888888888888884,
        "score": 37.886476797082956
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 40.158140045525016
      },
      "negation_pair_gen": {
       "accuracy": 0.36046511627906974,
       "cc": -13.691569202634321,
       "cc_by_type": {
        "choice": 23.333333333333332,
        "noul": -46.666666666666664,
        "score": -17.741374274569633
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.75,
       "cc": 64.2793105321989,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 80,
        "score": 48.873967632632734
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 64.2793105321989
      },
      "usecase": {
       "accuracy": 0.55,
       "cc": 32.18471870349252,
       "cc_by_type": {
        "choice": 53.43137254901962,
        "noul": 20.833333333333325,
        "score": 22.28945022812462
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 32.18471870349252
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 5.694428754991107,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 0,
        "score": 12.321381503068562
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 5.694428754991107
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 10.148014095788994,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -36.22262437929968
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 10.148014095788994
      },
      "de": {
       "accuracy": 0.75,
       "cc": 65.40389985072953,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 75,
        "score": 61.03470840174611
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 65.40389985072953
      },
      "el": {
       "accuracy": 0.5333333333333333,
       "cc": 38.05525802582019,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -33.333333333333336,
        "score": 97.77843701973244
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 38.05525802582019
      },
      "en": {
       "accuracy": 0.5708079930495221,
       "cc": 30.292533769567115,
       "cc_by_type": {
        "choice": 49.03650187329906,
        "noul": 16.546762589928065,
        "score": 25.29433684547422
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 30.292533769567115
      },
      "es": {
       "accuracy": 0.7096774193548387,
       "cc": 43.345191523632444,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 75,
        "score": -35.17649720724296
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 43.345191523632444
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 19.61940186440383,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 90.55219466424974
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 19.61940186440383
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 47.804264216420215,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 85.76573382573123
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 47.804264216420215
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": 7.619019958927157,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25,
        "score": -18.623387050592832
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 7.619019958927157
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8,
       "cc": 53.98762274993106,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -24.007513667587933
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 53.98762274993106
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 30.888991370728693,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 60.00000000000001,
        "score": 4.152279910438661
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 30.888991370728693
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 28.87078109529692,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 31.056787730335213
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 28.87078109529692
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.46731453674737,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.4019436102421
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.46731453674737
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 39.808499547935135,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 38.04021725852403
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 39.808499547935135
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.6923076923076923,
       "cc": 47.21002025003124,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 36.86493167688168
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 47.21002025003124
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 60.43669607801153,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 53.130948575814195
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 60.43669607801153
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": -0.9493715267830775,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60,
        "score": 59.27954499411885
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 38.182429440216794,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": -35.452711679349626
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 38.182429440216794
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 73.4457869308839,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 84.85348982490973
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 73.4457869308839
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 11.977084551443282,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -5.3116841987775
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 11.977084551443282
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.632,
       "cc": 52.46751755678448,
       "cc_by_type": {
        "choice": 52.46751755678448
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 52.46751755678448
      },
      "noul": {
       "accuracy": 0.5946666666666667,
       "cc": 18.933333333333337,
       "cc_by_type": {
        "noul": 18.933333333333337
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 18.933333333333337
      },
      "score": {
       "accuracy": 0.5173333333333333,
       "cc": 27.879918257891767,
       "cc_by_type": {
        "score": 27.879918257891767
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 27.879918257891767
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -39.93503064967874,
       "cc_by_type": {
        "choice": 20,
        "noul": -75,
        "score": -64.80509194903621
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 4.709594579392923,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -11.111111111111116,
        "score": -0.22594366002688382
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 4.709594579392923
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 68.93000085774041,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100,
        "score": 55.57049037809926
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 68.93000085774041
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -32.72560070376818,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": 7.086355783432308
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": 9.09103038460552,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 18.577438979903516
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 9.09103038460552
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -18.430913784732287,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -33.333333333333336,
        "score": -71.64494261205849
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 24.143316347420846,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -42.85714285714286,
        "score": 34.89493503666029
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 24.143316347420846
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 31.697117512192023,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 0,
        "score": 36.90319922647153
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 31.697117512192023
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 32.0762861885028,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": 60.00000000000001,
        "score": 40.395525232175046
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 32.0762861885028
      },
      "knowledge_graphs": {
       "accuracy": 0.45454545454545453,
       "cc": 24.991914766475592,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": 66.66666666666667,
        "score": -14.603127934691717
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 24.991914766475592
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 45.57097918413742,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": -7.214711026399101
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 45.57097918413742
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -28.76855468696155,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -33.333333333333336,
        "score": -88.45620169529325
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 41.46003313829909,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 19.999999999999996,
        "score": 27.412728973438515
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 41.46003313829909
      },
      "model_routing": {
       "accuracy": 0.7926267281105991,
       "cc": 64.71215206016127,
       "cc_by_type": {
        "choice": 69.61038961038962,
        "noul": 75.75757575757575,
        "score": 48.768490812518436
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 64.71215206016127
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 37.21060611300256,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": 0,
        "score": 57.69707745609023
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 37.21060611300256
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 45.50693100770957,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 19.900551611800964
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 45.50693100770957
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 14.94837838181516,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -50,
        "score": 39.18011051490362
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 14.94837838181516
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 38.068687261152014,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 38.843742942876325
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 38.068687261152014
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 90.14733599801389,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 81.11816457767156
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 90.14733599801389
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      68.27452416292749,
      76.49100458899656
     ],
     "I": [
      29.102270255987627,
      39.93644473899713
     ],
     "capability": [
      49.63071921143585,
      57.55172382456721
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 66
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 73.68644793817732,
     "capability": 54.52226717262408,
     "intelligence": 35.358086407070836
    },
    "per_type": {
     "choice": {
      "accuracy": 0.632,
      "calibration": 73.15681838850149,
      "cc": 52.46751755678448,
      "cc_by_type": {
       "choice": 52.46751755678448
      },
      "cc_tiered": 53.93148423434888,
      "cc_tiered_open": 57.132088899550936,
      "cc_tiered_sealed": 50.73087956914682,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 52.46751755678448
     },
     "noul": {
      "accuracy": 0.5946666666666667,
      "calibration": 69.54815278943805,
      "cc": 18.933333333333337,
      "cc_by_type": {
       "noul": 18.933333333333337
      },
      "cc_tiered": 19.54467770488492,
      "cc_tiered_open": 26.634230503795727,
      "cc_tiered_sealed": 12.45512490597411,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 18.933333333333337
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 78.35437263659244,
      "cc": 27.879918257891767,
      "cc_by_type": {
       "score": 27.879918257891767
      },
      "cc_tiered": 32.59809728197871,
      "cc_tiered_open": 42.092653594341826,
      "cc_tiered_sealed": 23.103540969615587,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.879918257891767
     }
    },
    "raw_on_A": {
     "calibration": 73.46402121876504,
     "capability": 55.14977186856087,
     "intelligence": 36.8355225183567
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "csabag",
   "axes": {
    "calibration": 87.41983564209619,
    "cost": 76.72111559464565,
    "intelligence": 25.185250272226448,
    "speed": 89.04172792696406
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07791307233311474,
      "ece_score": 84.41738553337706,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 84.41738553337706,
      "score_before_time_limit": 84.41738553337706,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.43466666666666665,
      "brier": 0.15904987256946293,
      "ece": 0.04699539774961242,
      "n": 375,
      "score": 90.60092045007751,
      "score_before_time_limit": 90.60092045007751,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.936,
      "interval90_width": 0.6326666666666665,
      "n": 375,
      "nrps": 0.12350841467172226,
      "score": 87.241200942834,
      "score_before_time_limit": 87.70898218113338,
      "time_limit_failed_share": 0.005333333333333333,
      "top_ece": 0.06115597085280505
     }
    },
    "score": 87.41983564209619
   },
   "capability": 56.30254295716132,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 902,192 input + 0 output tokens over 1,485 decisions = 607.536700337 tokens/decision. GPU-time cost USD 0.00596984167594128/1,000; old list-price cost USD 0.018226101010101/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00596984167594128/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.04166444
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005969841675941275
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 2,
    "n_over_limit_answered_ok": 2,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "RYOTIDE-Qwen (csabag, Qwen3.5-4B stock weights)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's own server, next-token softmax over the option markers after 'Answer: **', T 1.0, thinking off.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 27.26180765874407,
    "I_sealed": 23.10869288570883,
    "base": 25.185250272226448,
    "excess": -5.686411997320473,
    "gap": 4.153114773035238,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 52.57893858031065,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 45.265806857502355,
       "judge": 60.44825313117996,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": -4.347826086956519,
       "judge": -19.999999999999996,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 29.206484395921564,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.69798528911967,
       "hard": 25.394695936235557,
       "judge": 28.92308053498359,
       "standard": 23.50941666010148
      }
     },
     "sealed|choice": {
      "cc": 50.81225273237945,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 40.67306580316872,
       "judge": 55.04407443682666,
       "standard": 53.4908247205231
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 13.793103448275868,
       "hard": -3.6363636363636376,
       "judge": -23.076923076923073,
       "standard": 19.402985074626855
      }
     },
     "sealed|score": {
      "cc": 18.51382592474705,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 28.34102374356975,
       "hard": 15.694296995745871,
       "judge": 14.062136197193343,
       "standard": 25.916819464668606
      }
     }
    }
   },
   "jevbench_score": 13.458140852432997,
   "key": "ryotide-qwen",
   "last_measured_on": "2026-10-16",
   "licence": "per base model (stock weights; code licence not checked)",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "csabag/ryotide@6bbb615a170dc16b83ca62373aa7ada7bf3ea27e",
   "noul_decisive": {
    "acc_among_decisive": 0.8915094339622641,
    "decisive": 212,
    "decisive_rate": 0.5653333333333334,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.57576272152932,
     "cost": 62.17918681512502,
     "intelligence": 25.4219238453348,
     "speed": 89.1284523402127
    },
    "board_release": "v1.6.43",
    "capability": 56.49884328343206,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 77,
    "capability_rank": 103,
    "composite_ci95": {
     "A": [
      9.09465170059667,
      19.858266669161868
     ],
     "B": [
      7.381986198377254,
      16.979308525255792
     ],
     "C": [
      6.315730347636576,
      13.79046296469574
     ]
    },
    "jevbench_score": 13.247205181296051,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 123,
    "rank": 162,
    "ranks": {
     "A": 162,
     "B": 165,
     "C": 163,
     "capability": 158
    },
    "scores": {
     "A": 13.247205181296051,
     "B": 11.010415184681706,
     "C": 9.199448042566702
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.34223204720765354,
     "p50_s_raw": 0.09611602360382676,
     "p95_s_adjusted": 0.35713637880980964,
     "p95_s_raw": 0.10356818940490484
    },
    "views": {
     "Balanced 33:33:33": 11.63786334339517,
     "Emphasis on Accuracy 60:20:20": 8.895067587552155,
     "Emphasis on Cost 20:20:60": 13.082004734747406,
     "Emphasis on Speed 20:60:20": 14.510305108061395,
     "Intelligence only": 6.571813520246787
    }
   },
   "rank": 210,
   "ranked": true,
   "ranks": {
    "A": 210,
    "B": 210,
    "C": 210,
    "capability": 192
   },
   "repo": "https://github.com/csabag/ryotide",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 13.458140852432997,
    "B": 11.020179059435147,
    "C": 9.345931147522917
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3434839221648872,
    "p50_s_raw": 0.0967419610824436,
    "p95_s_adjusted": 0.36301184310577805,
    "p95_s_raw": 0.10650592155288904
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7272727272727273,
       "cc": 60.25604896413942,
       "cc_by_type": {
        "choice": 50,
        "noul": 100,
        "score": 30.768146892418248
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 60.25604896413942
      },
      "calibration_gen": {
       "accuracy": 0.46534653465346537,
       "cc": 10.945977140386304,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": -12,
        "score": -4.260689999837908
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 10.945977140386304
      },
      "composite_gen": {
       "accuracy": 0.43564356435643564,
       "cc": 0.8904059620089105,
       "cc_by_type": {
        "choice": 38.211382113821145,
        "noul": -48.14814814814815,
        "score": 12.607983920353739
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0.8904059620089105
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 82.45595791115925,
       "cc_by_type": {
        "choice": 100,
        "noul": 86.66666666666667,
        "score": 60.70120706681104
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 82.45595791115925
      },
      "jaggedness": {
       "accuracy": 0.3277777777777778,
       "cc": -10.87940523385924,
       "cc_by_type": {
        "choice": 23.484119345524558,
        "noul": -67.56756756756756,
        "score": 11.445232520465288
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.7092651757188498,
       "cc": 54.43283875070488,
       "cc_by_type": {
        "choice": 59.584397288016206,
        "noul": 64.94845360824742,
        "score": 38.76566535585102
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 54.43283875070488
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -102.05059677779968,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100,
        "score": -202.92598388178612
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5555555555555556,
       "cc": 17.436943706411615,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": -25,
        "score": 14.424233181090518
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 17.436943706411615
      },
      "multilingual": {
       "accuracy": 0.5854545454545454,
       "cc": 29.1324741350392,
       "cc_by_type": {
        "choice": 59.895663514835356,
        "noul": -8.333333333333337,
        "score": 35.83509222361558
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 29.1324741350392
      },
      "negation_pair_gen": {
       "accuracy": 0.3372093023255814,
       "cc": -21.488753564713154,
       "cc_by_type": {
        "choice": 26.666666666666668,
        "noul": -86.66666666666667,
        "score": -4.466260694139468
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.71875,
       "cc": 58.98116200821226,
       "cc_by_type": {
        "choice": 60.96096096096097,
        "noul": 60.00000000000001,
        "score": 55.9825250636758
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 58.98116200821226
      },
      "usecase": {
       "accuracy": 0.45,
       "cc": 15.363822315588715,
       "cc_by_type": {
        "choice": 46.07843137254903,
        "noul": -12.5,
        "score": 12.513035574217113
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 15.363822315588715
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 33.21333936912203,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50,
        "score": -31.31236284501484
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 33.21333936912203
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.38461538461538464,
       "cc": 2.2519380465291072,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -33.333333333333336,
        "score": 2.3011290397409345
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 2.2519380465291072
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 73.16070917811867,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 75,
        "score": 52.44672930426751
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 73.16070917811867
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 26.27536195081001,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 95.77208212803524
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 26.27536195081001
      },
      "en": {
       "accuracy": 0.526498696785404,
       "cc": 21.683070992286087,
       "cc_by_type": {
        "choice": 48.37749112166067,
        "noul": -1.4388489208633115,
        "score": 18.110570776060907
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 21.683070992286087
      },
      "es": {
       "accuracy": 0.9032258064516129,
       "cc": 75.28442317546406,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 35.641197748251905
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 75.28442317546406
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 18.43017965214669,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 54.19764278157668
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 18.43017965214669
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 35.04241420570067,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0,
        "score": 68.65665438180788
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 35.04241420570067
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 33.036921416788594,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -25,
        "score": 40.87054078667863
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 33.036921416788594
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 77.12684104141941,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 45.41014120687712
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 77.12684104141941
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 24.835101177285324,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 19.999999999999996,
        "score": 25.990609330108562
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 24.835101177285324
      },
      "ko": {
       "accuracy": 0.2,
       "cc": -25.044405505442313,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": 13.755672372561945
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 70.75737855456181,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 95.6054689970188
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 70.75737855456181
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 31.922090519792775,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 14.380990174096942
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 31.922090519792775
      },
      "no": {
       "accuracy": 0.5,
       "cc": 25,
       "cc_by_type": {
        "choice": 100,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 25
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 32.62285198021406,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 0,
        "score": 20.720862330824154
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 32.62285198021406
      },
      "pt": {
       "accuracy": 0.5714285714285714,
       "cc": 16.92560211002836,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": -19.999999999999996,
        "score": 23.811573566384435
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 16.92560211002836
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 20.77732952898762,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -19.999999999999996,
        "score": 33.39581837419692
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 20.77732952898762
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -7.370688274213745,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": 0,
        "score": -13.416412648728194
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 69.22817544196927,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 72.20065535816586
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 69.22817544196927
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 2.656257350195432,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -33.27416580252105
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 2.656257350195432
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.624,
       "cc": 51.434202721062405,
       "cc_by_type": {
        "choice": 51.434202721062405
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 51.434202721062405
      },
      "noul": {
       "accuracy": 0.504,
       "cc": 0.8000000000000007,
       "cc_by_type": {
        "noul": 0.8000000000000007
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0.8000000000000007
      },
      "score": {
       "accuracy": 0.43733333333333335,
       "cc": 21.804024593308835,
       "cc_by_type": {
        "score": 21.804024593308835
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 21.804024593308835
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -9.12561345211082,
       "cc_by_type": {
        "choice": 52,
        "noul": -50,
        "score": -29.37684035633246
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": -0.749620207485151,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": -55.55555555555556,
        "score": -28.059764694229084
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 55.49853707889798,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 49.015936439945975
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 55.49853707889798
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -39.558251355212086,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -36.803409036396495
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 12.48460978682998,
       "cc_by_type": {
        "choice": 27.53623188405797,
        "noul": -33.333333333333336,
        "score": 43.25093080976531
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 12.48460978682998
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -32.7199888811053,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -100,
        "score": -47.845501234510856
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 34.99630093396028,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 20.114953222048914
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 34.99630093396028
      },
      "gaming": {
       "accuracy": 0.22727272727272727,
       "cc": -24.636887531634997,
       "cc_by_type": {
        "choice": 5.9233449477351945,
        "noul": -100,
        "score": 20.165992457359817
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 40.74698882295959,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 60.00000000000001,
        "score": 24.740966468878767
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 40.74698882295959
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": -0.6958056755951295,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": -33.333333333333336,
        "score": 8.333710739096123
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 16.687683112704796,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -33.333333333333336,
        "score": -4.9757103518080825
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 16.687683112704796
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -23.05271961313592,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -4.642029807149695
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.4444444444444444,
       "cc": 12.971072584519268,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -19.999999999999996,
        "score": 16.494791649910965
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 12.971072584519268
      },
      "model_routing": {
       "accuracy": 0.7373271889400922,
       "cc": 55.23694097421215,
       "cc_by_type": {
        "choice": 71.94805194805195,
        "noul": 72.72727272727273,
        "score": 21.035498247311768
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 55.23694097421215
      },
      "moderation_trust_safety": {
       "accuracy": 0.3888888888888889,
       "cc": -12.70810171895986,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -100,
        "score": 30.97358351874422
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.5555555555555556,
       "cc": 42.400041022365116,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 27.29297357777317
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 42.400041022365116
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": 6.474201484263328,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50,
        "score": 28.535904945400816
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 6.474201484263328
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": 16.615962082396752,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": 33.33333333333333,
        "score": 3.4710746529873604
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 16.615962082396752
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 84.82622615418335,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 65.15483504617995
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 84.82622615418335
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.88139116603064,
      88.02962716518056
     ],
     "I": [
      21.652739997621005,
      29.685027913069895
     ],
     "capability": [
      52.06767897446016,
      57.896342508836945
     ],
     "composite": null,
     "seed": 67
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.41983564209619,
     "capability": 56.30254295716132,
     "intelligence": 25.185250272226448
    },
    "per_type": {
     "choice": {
      "accuracy": 0.624,
      "calibration": 84.41738553337706,
      "cc": 51.434202721062405,
      "cc_by_type": {
       "choice": 51.434202721062405
      },
      "cc_tiered": 51.69559565634505,
      "cc_tiered_open": 52.57893858031065,
      "cc_tiered_sealed": 50.81225273237945,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 51.434202721062405
     },
     "noul": {
      "accuracy": 0.504,
      "calibration": 90.60092045007751,
      "cc": 0.8000000000000007,
      "cc_by_type": {
       "noul": 0.8000000000000007
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0.8000000000000007
     },
     "score": {
      "accuracy": 0.43733333333333335,
      "calibration": 87.241200942834,
      "cc": 21.804024593308835,
      "cc_by_type": {
       "score": 21.804024593308835
      },
      "cc_tiered": 23.86015516033431,
      "cc_tiered_open": 29.206484395921564,
      "cc_tiered_sealed": 18.51382592474705,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 21.804024593308835
     }
    },
    "raw_on_A": {
     "calibration": 84.62485148494483,
     "capability": 55.81766663420327,
     "intelligence": 27.010481783461714
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 11.898514629288064,
    "Emphasis on Accuracy 60:20:20": 8.8476339446343,
    "Emphasis on Cost 20:20:60": 14.089358899953877,
    "Emphasis on Speed 20:60:20": 14.67734914335651,
    "Intelligence only": 6.389969777013225
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Csaba Garay",
   "axes": {
    "calibration": 77.35233849488566,
    "cost": 66.10615580655393,
    "intelligence": 27.627554756423923,
    "speed": 86.3537180778988
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12841215225172864,
      "ece_score": 74.31756954965427,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 322,
      "n_tvd": 0,
      "score": 72.83121815866117,
      "score_before_time_limit": 74.31756954965427,
      "time_limit_failed_share": 0.02
     },
     "noul": {
      "abstention_rate": 0.44266666666666665,
      "brier": 0.15244666314470043,
      "ece": 0.08234546671425014,
      "n": 375,
      "score": 83.08540848831184,
      "score_before_time_limit": 83.53090665714997,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9173333333333333,
      "interval90_width": 0.4935555555555556,
      "n": 375,
      "nrps": 0.13068295844559996,
      "score": 76.14038883768394,
      "score_before_time_limit": 76.9613094720525,
      "time_limit_failed_share": 0.010666666666666666,
      "top_ece": 0.1650454260566749
     }
    },
    "score": 77.35233849488566
   },
   "capability": 52.48994662565479,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 901,966 input + 0 output tokens over 1,479 decisions = 609.848546315 tokens/decision. GPU-time cost USD 0.0134832568106899/1,000; old list-price cost USD 0.0487878837052062/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0134832568106899/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.13612464
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01348325681068992
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 21,
    "n_over_limit_answered_ok": 21,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "RYOTIDE-Qwen9",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 39.61561448217045,
    "I_sealed": 18.921235479400952,
    "base": 29.268424980785703,
    "excess": 13.606281258520013,
    "gap": 20.694379002769498,
    "penalty": 0.9439371874147998,
    "per_type_split": {
     "open|choice": {
      "cc": 62.49558303248294,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 68.86486486486487,
       "hard": 56.59012268008808,
       "judge": 68.35860250494397,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 26.282953761214635,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 30.06830665281378,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 39.087811240948376,
       "hard": 37.67076473034441,
       "judge": 33.89329590307768,
       "standard": 4.616154328289368
      }
     },
     "sealed|choice": {
      "cc": 47.768113114558034,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 72.21936413211235,
       "hard": 39.22341024198085,
       "judge": 49.8861047835991,
       "standard": 49.45490584737365
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 25,
       "hard": -24.13793103448276,
       "judge": -16.417910447761198,
       "standard": 13.043478260869556
      }
     },
     "sealed|score": {
      "cc": 8.995593323644819,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 35.843219543729575,
       "hard": 1.8186910992705863,
       "judge": -2.3511609711775794,
       "standard": 26.945716104584506
      }
     }
    }
   },
   "jevbench_score": 16.10488822146105,
   "key": "ryotide_qwen9",
   "last_measured_on": "2026-10-10",
   "licence": "MIT engine on Apache-2.0 weights",
   "listing": "ranked",
   "listing_note": "Reclassified 12 Oct 2026 from wrapper to native ranked: a training-free typed-decision readout over unmodified open weights (Qwen/Qwen3.5-9B, revision c2022362), like the other native open-weights systems, not a service running another entrant's model. v1.6.4 scores, axes and cost unchanged; it was never part of the four-native v1.6.3 field median and does not enter it.",
   "measured_in": "v1.6.4",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "c202236235762e1c871ad0ccb60c8ee5ba337b9a",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.2699285596189,
     "cost": 49.35064060220574,
     "intelligence": 28.676598568644657,
     "speed": 86.71260028928383
    },
    "board_release": "v1.6.43",
    "capability": 53.473263564131784,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 88,
    "capability_rank": 114,
    "composite_ci95": {
     "A": [
      10.301952384081664,
      20.49332468149985
     ],
     "B": [
      8.686711300573288,
      18.11978988063779
     ],
     "C": [
      7.154133600056712,
      14.23147547326378
     ]
    },
    "jevbench_score": 16.1347687863636,
    "measured_in": "v1.6.4",
    "method": "v1.6",
    "open_board_rank": 115,
    "rank": 153,
    "ranks": {
     "A": 153,
     "B": 153,
     "C": 153,
     "capability": 169
    },
    "scores": {
     "A": 16.1347687863636,
     "B": 14.016103437178304,
     "C": 11.204700546085835
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4132151529999192,
     "p50_s_raw": 0.13160757649995958,
     "p95_s_adjusted": 0.5158978665999938,
     "p95_s_raw": 0.18294893329999692
    },
    "views": {
     "Balanced 33:33:33": 14.420164341278662,
     "Emphasis on Accuracy 60:20:20": 11.745818244481526,
     "Emphasis on Cost 20:20:60": 14.947291611405522,
     "Emphasis on Speed 20:60:20": 17.856002324382263,
     "Intelligence only": 9.189428059162678
    }
   },
   "rank": 201,
   "ranked": true,
   "ranks": {
    "A": 201,
    "B": 202,
    "C": 201,
    "capability": 204
   },
   "repo": "https://github.com/csabag/ryotide/tree/94c71a3d9f044ffaba87981e985b66f47de3be80",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.10488822146105,
    "B": 13.626769396844377,
    "C": 11.183950153792397
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.41632815200002826,
    "p50_s_raw": 0.13316407600001412,
    "p95_s_adjusted": 0.5561506042999099,
    "p95_s_raw": 0.20307530214995495
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.76,
       "cc": 66.05136415304769,
       "cc_by_type": {
        "choice": 64.44444444444444,
        "noul": 50,
        "score": 83.70964801469863
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 66.05136415304769
      },
      "calibration_gen": {
       "accuracy": 0.4479166666666667,
       "cc": 14.44148530317438,
       "cc_by_type": {
        "choice": 42.38683127572016,
        "noul": 33.33333333333333,
        "score": -32.395708699530346
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 14.44148530317438
      },
      "composite_gen": {
       "accuracy": 0.37,
       "cc": -19.57376093365861,
       "cc_by_type": {
        "choice": 47.96747967479675,
        "noul": -68,
        "score": -38.68876247577258
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.7142857142857143,
       "cc": 71.76642296515836,
       "cc_by_type": {
        "choice": 79.66101694915254,
        "noul": 100,
        "score": 35.63825194632254
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 71.76642296515836
      },
      "jaggedness": {
       "accuracy": 0.3888888888888889,
       "cc": -12.593176711081462,
       "cc_by_type": {
        "choice": 33.15177024015846,
        "noul": -85.71428571428572,
        "score": 14.782985340882881
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.7207792207792207,
       "cc": 56.929464534321276,
       "cc_by_type": {
        "choice": 59.86801034513512,
        "noul": 52.08333333333333,
        "score": 58.83704992449537
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 56.929464534321276
      },
      "long_state": {
       "accuracy": 0,
       "cc": -131.99233716475095,
       "cc_by_type": {
        "choice": -29.310344827586214,
        "noul": -100,
        "score": -266.66666666666663
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4411764705882353,
       "cc": 0.9235219934698241,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -50,
        "score": 15.92846071725158
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 0.9235219934698241
      },
      "multilingual": {
       "accuracy": 0.5859649122807018,
       "cc": 32.2795517679191,
       "cc_by_type": {
        "choice": 50.49813200498132,
        "noul": 9.756097560975618,
        "score": 36.584425737800366
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 32.2795517679191
      },
      "negation_pair_gen": {
       "accuracy": 0.3979591836734694,
       "cc": -11.709890170422623,
       "cc_by_type": {
        "choice": 43.33333333333332,
        "noul": -41.17647058823529,
        "score": -37.28653325636591
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.6666666666666666,
       "cc": 57.532861476238615,
       "cc_by_type": {
        "choice": 57.532861476238615
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 57.532861476238615
      },
      "perturb_pair": {
       "accuracy": 0.7419354838709677,
       "cc": 45.73965152932314,
       "cc_by_type": {
        "choice": 71.69811320754718,
        "noul": 16.666666666666675,
        "score": 48.85417471375556
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 45.73965152932314
      },
      "usecase": {
       "accuracy": 0.46078431372549017,
       "cc": 14.920533192964271,
       "cc_by_type": {
        "choice": 52.203136669156095,
        "noul": -24,
        "score": 16.558462909736715
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 14.920533192964271
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.47058823529411764,
       "cc": 15.197907909052608,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0,
        "score": 2.7365808700149707
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 15.197907909052608
      },
      "cs": {
       "accuracy": 0.7777777777777778,
       "cc": 56.32183908045977,
       "cc_by_type": {
        "choice": 79.3103448275862,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 56.32183908045977
      },
      "da": {
       "accuracy": 0.6875,
       "cc": 14.572239106386718,
       "cc_by_type": {
        "choice": 84.44252376836647,
        "noul": 0,
        "score": -40.72580644920632
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 14.572239106386718
      },
      "de": {
       "accuracy": 0.7,
       "cc": 60.32572841250546,
       "cc_by_type": {
        "choice": 55.53725813091807,
        "noul": 63.63636363636365,
        "score": 61.80356347023468
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 60.32572841250546
      },
      "el": {
       "accuracy": 0.5294117647058824,
       "cc": 27.556367507464582,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": -19.999999999999996,
        "score": 59.81195966525089
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 27.556367507464582
      },
      "en": {
       "accuracy": 0.5259911894273128,
       "cc": 18.612787223400975,
       "cc_by_type": {
        "choice": 50.18367752184269,
        "noul": -5.223880597014929,
        "score": 10.878564745375174
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 18.612787223400975
      },
      "es": {
       "accuracy": 0.8333333333333334,
       "cc": 69.61549652183463,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 63.63636363636365,
        "score": 45.21012592914022
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 69.61549652183463
      },
      "fi": {
       "accuracy": 0.42857142857142855,
       "cc": -0.0686132800628864,
       "cc_by_type": {
        "choice": 23.076923076923077,
        "noul": -100,
        "score": 76.71723708288826
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5625,
       "cc": 12.535202989271738,
       "cc_by_type": {
        "choice": 42.16867469879518,
        "noul": -100,
        "score": 95.43693426902003
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 12.535202989271738
      },
      "hi": {
       "accuracy": 0.5,
       "cc": 1.3565370500971834,
       "cc_by_type": {
        "choice": 44.547134935304996,
        "noul": -33.333333333333336,
        "score": -7.14419045168011
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 1.3565370500971834
      },
      "id": {
       "accuracy": 0.5,
       "cc": 34.42622950819673,
       "cc_by_type": {
        "choice": 34.42622950819673
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 34.42622950819673
      },
      "it": {
       "accuracy": 0.7692307692307693,
       "cc": 55.70093751575863,
       "cc_by_type": {
        "choice": 83.98576512455516,
        "noul": 33.33333333333333,
        "score": 49.78371408938743
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 55.70093751575863
      },
      "ja": {
       "accuracy": 0.45454545454545453,
       "cc": -6.061268823867529,
       "cc_by_type": {
        "choice": 43.172849250197324,
        "noul": 0,
        "score": -61.35665572179991
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.4444444444444444,
       "cc": -4.546760795957428,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": -100,
        "score": 39.69305094546105
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9333333333333333,
       "cc": 91.97613585457272,
       "cc_by_type": {
        "choice": 77.77777777777779,
        "noul": 100,
        "score": 98.1506297859404
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 91.97613585457272
      },
      "nl": {
       "accuracy": 0.4117647058823529,
       "cc": -1.0752965494261122,
       "cc_by_type": {
        "choice": 13.738019169329075,
        "noul": 0,
        "score": -16.96390881760741
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 29.411861798854904,
       "cc_by_type": {
        "choice": 69.84924623115577,
        "noul": 16.666666666666675,
        "score": 1.719672498742264
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 29.411861798854904
      },
      "pt": {
       "accuracy": 0.5833333333333334,
       "cc": 44.85538134603173,
       "cc_by_type": {
        "choice": 36.530324400564176,
        "noul": 50,
        "score": 48.03581963753102
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 44.85538134603173
      },
      "sv": {
       "accuracy": 0.25,
       "cc": -2.8689134454598872,
       "cc_by_type": {
        "choice": 6.7357512953367875,
        "noul": -33.333333333333336,
        "score": 17.990841701616887
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.6363636363636364,
       "cc": 8.662553165653643,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": -30.534079633473855
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 8.662553165653643
      },
      "uk": {
       "accuracy": 0.6666666666666666,
       "cc": 50.95552616234304,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 84.04937418595387
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 50.95552616234304
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -18.2554748281775,
       "cc_by_type": {
        "choice": 36.170212765957444,
        "noul": 33.33333333333333,
        "score": -124.26997058382328
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.62,
       "cc": 50.80458006482671,
       "cc_by_type": {
        "choice": 50.80458006482671
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 50.80458006482671
      },
      "noul": {
       "accuracy": 0.5066666666666667,
       "cc": 1.333333333333342,
       "cc_by_type": {
        "noul": 1.333333333333342
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 1.333333333333342
      },
      "score": {
       "accuracy": 0.432,
       "cc": 16.638727413057563,
       "cc_by_type": {
        "score": 16.638727413057563
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 16.638727413057563
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4,
       "cc": -0.33698217915645756,
       "cc_by_type": {
        "choice": 46.996466431095406,
        "noul": -50,
        "score": 1.9925870314352223
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4090909090909091,
       "cc": 6.182899746566381,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -19.999999999999996,
        "score": -5.550679642288436
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 6.182899746566381
      },
      "customer_support": {
       "accuracy": 0.5384615384615384,
       "cc": 22.1821554104641,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0,
        "score": 31.763857535740126
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 22.1821554104641
      },
      "demand_forecasting": {
       "accuracy": 0.2857142857142857,
       "cc": -30.31949857431742,
       "cc_by_type": {
        "choice": 9.090909090909092,
        "noul": -100,
        "score": -0.049404813861353425
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5,
       "cc": 25.651052971843274,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 19.999999999999996,
        "score": 22.526929407333107
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 25.651052971843274
      },
      "feature_extraction": {
       "accuracy": 0.4375,
       "cc": -20.811031444586995,
       "cc_by_type": {
        "choice": 21.875,
        "noul": -100,
        "score": 15.691905666239014
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 36.86432069029883,
       "cc_by_type": {
        "choice": 57.44680851063829,
        "noul": 19.999999999999996,
        "score": 33.14615356025822
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 36.86432069029883
      },
      "gaming": {
       "accuracy": 0.29411764705882354,
       "cc": -24.720180964179843,
       "cc_by_type": {
        "choice": 15.69086651053864,
        "noul": -100,
        "score": 10.148590596921835
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 18.69117622868764,
       "cc_by_type": {
        "choice": 21.052631578947366,
        "noul": 0,
        "score": 35.02089710711556
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 18.69117622868764
      },
      "knowledge_graphs": {
       "accuracy": 0.3333333333333333,
       "cc": -1.336281990839917,
       "cc_by_type": {
        "choice": 20.353982300884958,
        "noul": -42.85714285714286,
        "score": 18.49431458373815
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.5925925925925926,
       "cc": 34.86130124669457,
       "cc_by_type": {
        "choice": 79.79797979797979,
        "noul": 0,
        "score": 24.785923942103917
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 34.86130124669457
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 5.407718729482173,
       "cc_by_type": {
        "choice": 100,
        "noul": -75,
        "score": -8.77684381155348
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 5.407718729482173
      },
      "llm_guardrails": {
       "accuracy": 0.5416666666666666,
       "cc": 21.13264329944355,
       "cc_by_type": {
        "choice": 50.81967213114754,
        "noul": -14.28571428571429,
        "score": 26.8639720528974
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 21.13264329944355
      },
      "model_routing": {
       "accuracy": 0.7881773399014779,
       "cc": 64.2670663183282,
       "cc_by_type": {
        "choice": 73.78004369992718,
        "noul": 76.27118644067797,
        "score": 42.74996881437948
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 64.2670663183282
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 60.10650947754033,
       "cc_by_type": {
        "choice": 41.55844155844156,
        "noul": 66.66666666666667,
        "score": 72.09442020751275
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 60.10650947754033
      },
      "recruiting": {
       "accuracy": 0.5294117647058824,
       "cc": 13.055323558022193,
       "cc_by_type": {
        "choice": 81.19122257053293,
        "noul": -50,
        "score": 7.9747481035336465
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 13.055323558022193
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": 5.36471360451591,
       "cc_by_type": {
        "choice": 55.99022004889975,
        "noul": -33.333333333333336,
        "score": -6.562745902018685
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 5.36471360451591
      },
      "scientific_discovery": {
       "accuracy": 0.38095238095238093,
       "cc": -0.6081023002725369,
       "cc_by_type": {
        "choice": 27.18446601941747,
        "noul": -50,
        "score": 20.99122707976492
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.75,
       "cc": 46.972150398582045,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": -19.08354880425387
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 46.972150398582045
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.06512048464919,
      79.83352815497564
     ],
     "I": [
      23.25648858171055,
      30.89797430499399
     ],
     "capability": [
      48.41837842556112,
      54.43007105303262
     ],
     "composite": null,
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.35233849488566,
     "capability": 52.48994662565479,
     "intelligence": 27.627554756423923
    },
    "per_type": {
     "choice": {
      "accuracy": 0.62,
      "calibration": 72.83121815866117,
      "cc": 50.80458006482671,
      "cc_by_type": {
       "choice": 50.80458006482671
      },
      "cc_tiered": 55.13184807352049,
      "cc_tiered_open": 62.49558303248294,
      "cc_tiered_sealed": 47.768113114558034,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 50.80458006482671
     },
     "noul": {
      "accuracy": 0.5066666666666667,
      "calibration": 83.08540848831184,
      "cc": 1.333333333333342,
      "cc_by_type": {
       "noul": 1.333333333333342
      },
      "cc_tiered": 13.141476880607318,
      "cc_tiered_open": 26.282953761214635,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 1.333333333333342
     },
     "score": {
      "accuracy": 0.432,
      "calibration": 76.14038883768394,
      "cc": 16.638727413057563,
      "cc_by_type": {
       "score": 16.638727413057563
      },
      "cc_tiered": 19.5319499882293,
      "cc_tiered_open": 30.06830665281378,
      "cc_tiered_sealed": 8.995593323644819,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 16.638727413057563
     }
    },
    "raw_on_A": {
     "calibration": 77.84867467337358,
     "capability": 54.81455349282751,
     "intelligence": 31.78043231228143
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.020000000000000018,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.561071682759692,
    "Emphasis on Accuracy 60:20:20": 11.2832469640574,
    "Emphasis on Cost 20:20:60": 16.386896276886198,
    "Emphasis on Speed 20:60:20": 17.737587769876722,
    "Intelligence only": 8.43504368871633
   }
  },
  {
   "addendum": null,
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "Levanto Labs",
   "axes": {
    "calibration": 91.59817606608073,
    "cost": 58.231203787388296,
    "intelligence": 65.59290944525489,
    "speed": 91.59383325786439
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.034436035156249975,
      "ece_score": 93.11279296875,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 93.11279296875,
      "score_before_time_limit": 93.11279296875,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.16533333333333333,
      "brier": 0.08087898763020833,
      "ece": 0.049375,
      "n": 375,
      "score": 90.125,
      "score_before_time_limit": 90.125,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.968,
      "interval90_width": 0.2808888888888889,
      "n": 375,
      "nrps": 0.06392779541015624,
      "score": 91.55673522949218,
      "score_before_time_limit": 91.55673522949218,
      "time_limit_failed_share": 0,
      "top_ece": 0.05246875
     }
    },
    "score": 91.59817606608073
   },
   "capability": 78.59554275566781,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     71.4032274840039,
     74.84235710953574
    ],
    "B": [
     68.73566282385947,
     73.77740634012592
    ],
    "C": [
     71.0269206102275,
     74.84235710953574
    ]
   },
   "cost": {
    "basis": "Levanto list tariff, top tier: USD 0.05 per 1M input tokens, USD 10 per 1M output tokens (https://levanto.ai/pricing, read 5 Oct 2026)",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.06497606666666667
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.024677048070413
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Sage 1.3.0 (Levanto Labs)",
   "endpoint_condition": "Levanto hosted API, text only, measured from Sandy (Helsinki) on 5 Oct 2026; full set 1,200 sealed + P300 (v1.6.1); not evaluated on ImageJevBench.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 65.67742817331194,
    "I_sealed": 65.50839071719783,
    "base": 65.59290944525489,
    "excess": -2.4043501863241326,
    "gap": 0.16903745611411125,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 85.05619838199217,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 83.01352626612143,
       "judge": 84.17930125247197,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 54.291235334713605,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 56.52173913043479,
       "judge": 39.99999999999999,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 57.68485080323005,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 71.80397727272727,
       "hard": 52.14120370370371,
       "judge": 68.19385593220339,
       "standard": 45.94907407407408
      }
     },
     "sealed|choice": {
      "cc": 79.68397614698543,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 82.70637794693668,
       "hard": 73.02036199095022,
       "judge": 85.1024208566108,
       "standard": 83.37233649464221
      }
     },
     "sealed|noul": {
      "cc": 56.36040792532405,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": 41.747572815533985,
       "judge": 57.89473684210527,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 60.480788079284,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 76.96974734042553,
       "hard": 51.94546568627452,
       "judge": 64.89053143712576,
       "standard": 62.692338197969534
      }
     }
    }
   },
   "jevbench_score": 73.72797965992304,
   "key": "sage-1.3.0",
   "last_measured_on": "2026-10-05",
   "licence": "proprietary hosted API",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "levanto-sage-v1.3 (Sage 1.3.0)",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9488817891373802,
    "decisive_rate": 0.8346666666666667,
    "supported": true,
    "valid": 375
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 8,
    "axes": {
     "calibration": 91.59817606608073,
     "cost": 58.231203787388296,
     "intelligence": 65.59290944525489,
     "speed": 93.61638181239104
    },
    "board_release": "v1.6.43",
    "capability": 78.59554275566781,
    "capability_api_board_rank": 9,
    "capability_open_board_rank": null,
    "capability_rank": 17,
    "composite_ci95": {
     "A": [
      71.70514550946575,
      75.17412681523324
     ],
     "B": [
      68.95926995267655,
      74.03508105576502
     ],
     "C": [
      71.32573829825638,
      75.17412681523324
     ]
    },
    "jevbench_score": 74.04992179175468,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 9,
    "ranks": {
     "A": 9,
     "B": 10,
     "C": 9,
     "capability": 33
    },
    "scores": {
     "A": 74.04992179175468,
     "B": 72.1884444725083,
     "C": 74.04992179175468
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.15245595201849937,
     "p50_s_raw": 0.15245595201849937,
     "p95_s_adjusted": 0.2852446019649505,
     "p95_s_raw": 0.2852446019649505
    },
    "views": {
     "Balanced 33:33:33": 69.60497929786021,
     "Emphasis on Accuracy 60:20:20": 67.94266119076805,
     "Emphasis on Cost 20:20:60": 64.56093721161619,
     "Emphasis on Speed 20:60:20": 77.56249365186707,
     "Intelligence only": 65.59290944525489
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": 37,
   "ranked": true,
   "ranks": {
    "A": 37,
    "B": 38,
    "C": 32,
    "capability": 43
   },
   "repo": "https://levanto.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.72797965992304,
    "B": 71.94344239717388,
    "C": 73.72797965992304
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.17092861235141754,
    "p50_s_raw": 0.17092861235141754,
    "p95_s_adjusted": 0.4053236149251461,
    "p95_s_raw": 0.4053236149251461
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.55241460870812,
      92.09209592889854
     ],
     "I": [
      60.000783824692206,
      70.03012866456243
     ],
     "capability": [
      74.15252653755351,
      80.27098937533351
     ],
     "composite": {
      "A": [
       71.4032274840039,
       74.84235710953574
      ],
      "B": [
       68.73566282385947,
       73.77740634012592
      ],
      "C": [
       71.0269206102275,
       74.84235710953574
      ]
     },
     "seed": 114
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.59817606608073,
     "capability": 78.59554275566781,
     "intelligence": 65.59290944525489
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8493333333333334,
      "calibration": 93.11279296875,
      "cc": 80.73188522493676,
      "cc_by_type": {
       "choice": 80.73188522493676
      },
      "cc_tiered": 82.3700872644888,
      "cc_tiered_open": 85.05619838199217,
      "cc_tiered_sealed": 79.68397614698543,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 80.73188522493676
     },
     "noul": {
      "accuracy": 0.792,
      "calibration": 90.125,
      "cc": 58.400000000000006,
      "cc_by_type": {
       "noul": 58.400000000000006
      },
      "cc_tiered": 55.32582163001882,
      "cc_tiered_open": 54.291235334713605,
      "cc_tiered_sealed": 56.36040792532405,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 58.400000000000006
     },
     "score": {
      "accuracy": 0.68,
      "calibration": 91.55673522949218,
      "cc": 60.08634868421052,
      "cc_by_type": {
       "score": 60.08634868421052
      },
      "cc_tiered": 59.08281944125702,
      "cc_tiered_open": 57.68485080323005,
      "cc_tiered_sealed": 60.480788079284,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 60.08634868421052
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.22612697189255,
    "Emphasis on Accuracy 60:20:20": 67.72558799363198,
    "Emphasis on Cost 20:20:60": 64.36490392701504,
    "Emphasis on Speed 20:60:20": 76.72033411975153,
    "Intelligence only": 65.59290944525489
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Levanto Labs (marco@levanto.ai)",
   "axes": {
    "calibration": 82.91532974968945,
    "cost": 59.47868179714396,
    "intelligence": 63.700734406993774,
    "speed": 68.89876304791818
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06843028052805278,
      "ece_score": 86.31394389438944,
      "mean_tvd": null,
      "n": 738,
      "n_ece_hard": 303,
      "n_tvd": 0,
      "score": 85.16309130913092,
      "score_before_time_limit": 86.31394389438944,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.08847184986595169,
      "brier": 0.10158675861102966,
      "ece": 0.08940390415549597,
      "n": 373,
      "score": 81.4622654155496,
      "score_before_time_limit": 82.1192191689008,
      "time_limit_failed_share": 0.008
     },
     "score": {
      "interval90_coverage": 0.8897849462365591,
      "interval90_width": 0.2163978494623655,
      "n": 372,
      "nrps": 0.07738047774120042,
      "score": 82.12063252438782,
      "score_before_time_limit": 82.56095763175719,
      "time_limit_failed_share": 0.005333333333333333,
      "top_ece": 0.13570018481182797
     }
    },
    "score": 82.91532974968945
   },
   "capability": 73.30803207834161,
   "composite_ci95": {
    "A": [
     65.47342000127809,
     68.94694303660847
    ],
    "B": [
     63.90184880923016,
     68.72718774008524
    ],
    "C": [
     61.52814597282892,
     68.94694303660847
    ]
   },
   "cost": {
    "basis": "Levanto Labs public list price, USD 0.05/M input, USD 10/M output (fetched 16 Oct 2026): Levanto's public pricing page lists Input USD 0.05 per 1M tokens on its Free and Developer plans (USD 0.046, 0.042 and 0.038 only on the paid monthly volume plans) and \"All plans: Output $10 / 1M tokens\", where output tokens are Sage's reasoning and answers are free (saved copy sha256 1e98749c...). The page states one tariff for the Sage API and does not name a model version. It is the same list basis the published sage-1.3.0 row carries. Counted as the manufacturer's standard, non-promotional launch list price under interpretation I-1 (the reading that admitted kushal-gemma4-31b-it-autoloops, SubQ Snap and d1-ultra), through a scoped re-freeze of the frozen pricing module that adds exactly three keys (sage-1.4.0, nimble-v4, nimble-v4-max) to its launch-list-price set and nothing else, with a zero-difference proof for every other registry entry and the independent release review; it answers the pricing question that the published review left open. The Free plan's included usage is a trial credit and is not used. ESTIMATE in the scorer's sense: the tariff x this row's OWN provider-reported tokens (665,989 input tokens over the 1,483 answered items; the provider reported 0 output tokens on every answered item, so the output term is zero as measured), divided by the 1,485 items of the common cost basis the scorer uses; the 17 failed requests (15 HTTP 400, 2 HTTP 504) count as zero tokens and as wrong answers. USD 0.022424 per 1,000 decisions on the common basis (USD 0.022200 over all 1,500 items), 0.347x the official cost cap USD 0.064595. Not a bill and not a quote. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.022424 per 1,000 decisions (0.347x the official cost cap USD 0.064595), against USD 0.022200 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.022199633333333333
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.022423872053872054
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 13,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Sage 1.4 (Levanto Labs)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted sage.levanto.ai /v1/systemone API, model `levanto-sage-v1.4`; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `levanto` before the first request (every sealed item of this draw was already retired). Native probabilities on all three question types, no generated tokens. Hosting implementation and deployed weights were not independently inspected. Regular-queue request 5181ADBA; model id read from the vendor's own /v1/models. 15 of 1,500 requests returned HTTP 400 'Extended context is not available on your plan' with the submitter's own key; they are the draw's 15 longest items (the same 15 that Uniphore-4B-Decider, Attest-31B and Standard One 3B SH failed on). 2 more returned HTTP 504 gateway timeouts. The runner never re-sends a failed item, so all 17 are scored as wrong in the full denominator.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 66.7993471941146,
    "I_sealed": 60.60212161987294,
    "base": 63.700734406993774,
    "excess": -3.6423011961140475,
    "gap": 6.1972255742416635,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 80.09787615590605,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 77.35136835482857,
       "judge": 80.22412656558998,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 63.46997929606625,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 39.99999999999999,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 56.830186130371516,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 69.2471590909091,
       "hard": 63.693576388888886,
       "judge": 56.137447033898304,
       "standard": 37.93402777777778
      }
     },
     "sealed|choice": {
      "cc": 74.0921412438383,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 71.37735630854631,
       "judge": 68.26640548481882,
       "standard": 79.11832946635731
      }
     },
     "sealed|noul": {
      "cc": 60.500725214055116,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 75.86206896551724,
       "hard": 58.18181818181818,
       "judge": 60.00000000000001,
       "standard": 58.20895522388059
      }
     },
     "sealed|score": {
      "cc": 47.2134984017254,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 70.73396381578947,
       "hard": 39.79533450704228,
       "judge": 41.695149739583336,
       "standard": 58.56711647727272
      }
     }
    }
   },
   "jevbench_score": 67.69827866650344,
   "key": "sage-1.4.0",
   "last_measured_on": "2026-10-13",
   "licence": "Proprietary hosted API",
   "listing": "ranked",
   "measured_in": "v1.6.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:sage.levanto.ai/v1/systemone#levanto-sage-v1.4",
   "noul_decisive": {
    "acc_among_decisive": 0.9050445103857567,
    "decisive": 337,
    "decisive_rate": 0.9108108108108108,
    "n": 375,
    "supported": true,
    "valid": 370
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 19,
    "axes": {
     "calibration": 83.66470689834914,
     "cost": 59.47868179714396,
     "intelligence": 65.88206323551285,
     "speed": 68.80003106294234
    },
    "board_release": "v1.6.43",
    "capability": 74.77338506693098,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      66.41270016475069,
      69.58030491259257
     ],
     "B": [
      65.14310488978832,
      69.5878926976111
     ],
     "C": [
      66.25545115064749,
      69.58030491259257
     ]
    },
    "jevbench_score": 68.40093468849139,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 35,
    "ranks": {
     "A": 35,
     "B": 30,
     "C": 29,
     "capability": 46
    },
    "scores": {
     "A": 68.40093468849139,
     "B": 67.881868670303,
     "C": 68.40093468849139
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 1.7410984199959785,
     "p50_s_raw": 1.7410984199959785,
     "p95_s_adjusted": 7.571354350852194,
     "p95_s_raw": 7.571354350852194
    },
    "views": {
     "Balanced 33:33:33": 64.47971276851106,
     "Emphasis on Accuracy 60:20:20": 65.03342740838724,
     "Emphasis on Cost 20:20:60": 62.38166597812596,
     "Emphasis on Speed 20:60:20": 66.14105152371843,
     "Intelligence only": 65.88206323551285
    }
   },
   "rank": 98,
   "ranked": true,
   "ranks": {
    "A": 98,
    "B": 79,
    "C": 69,
    "capability": 74
   },
   "repo": "https://levanto.ai/",
   "repriced_in": "v1.6.23",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 67.69827866650344,
    "B": 66.85912876074042,
    "C": 67.69827866650344
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.705115354496229,
    "p50_s_raw": 1.705115354496229,
    "p95_s_adjusted": 7.5573568566484015,
    "p95_s_raw": 7.5573568566484015
   },
   "status": {
    "answered_ok": 1483,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 77.47395833333333,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 57.421875
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 77.47395833333333
      },
      "calibration_gen": {
       "accuracy": 0.594059405940594,
       "cc": 48.13921763156151,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 76,
        "score": 16.1376953125
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 48.13921763156151
      },
      "composite_gen": {
       "accuracy": 0.8118811881188119,
       "cc": 63.04812180066244,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 62.962962962962955,
        "score": 35.937499999999986
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 63.04812180066244
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 86.10966435185185,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 98.32899305555556
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 86.10966435185185
      },
      "jaggedness": {
       "accuracy": 0.6722222222222223,
       "cc": 44.68650078063118,
       "cc_by_type": {
        "choice": 66.79499518768047,
        "noul": 18.918918918918926,
        "score": 48.34558823529412
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 44.68650078063118
      },
      "llm_router": {
       "accuracy": 0.9201277955271565,
       "cc": 84.76639284962694,
       "cc_by_type": {
        "choice": 91.28290921898389,
        "noul": 97.93814432989691,
        "score": 65.07812500000001
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 84.76639284962694
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": 15.876531561937488,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -25,
        "score": 46.8563988095238
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 15.876531561937488
      },
      "multilingual": {
       "accuracy": 0.7236363636363636,
       "cc": 57.13059893713923,
       "cc_by_type": {
        "choice": 67.72089990218454,
        "noul": 44.44444444444444,
        "score": 59.22645246478873
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 57.13059893713923
      },
      "negation_pair_gen": {
       "accuracy": 0.8604651162790697,
       "cc": 63.684895833333336,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 31.0546875
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 63.684895833333336
      },
      "option_order": {
       "accuracy": 0.8958333333333334,
       "cc": 86.68442077230361,
       "cc_by_type": {
        "choice": 86.68442077230361
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 86.68442077230361
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 72.47917087760838,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 80,
        "score": 58.45853365384615
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 72.47917087760838
      },
      "usecase": {
       "accuracy": 0.7363636363636363,
       "cc": 62.40265522875817,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 62.5,
        "score": 51.668749999999996
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 62.40265522875817
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 52.014175061050054,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50,
        "score": 25.09014423076922
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 52.014175061050054
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 48.91221038146441,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 34.14062499999999
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 48.91221038146441
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 83.19280452417597,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100,
        "score": 81.4368206521739
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.19280452417597
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 61.01800124146493,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 33.33333333333333,
        "score": 100
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 61.01800124146493
      },
      "en": {
       "accuracy": 0.787141615986099,
       "cc": 65.27214325022268,
       "cc_by_type": {
        "choice": 78.47231544647977,
        "noul": 67.62589928057554,
        "score": 49.71821502361274
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 65.27214325022268
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 64.7293101069955,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 13.763786764705888
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 64.7293101069955
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 55.919854280510016,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 55.919854280510016
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 97.08059210526316,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 91.24177631578947
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 97.08059210526316
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 34.37563649324954,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": 25,
        "score": 45.166015625
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 34.37563649324954
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 67.14265706769847,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 15.457589285714269
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 67.14265706769847
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 38.88002217368282,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 39.99999999999999,
        "score": 19.53125000000001
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 38.88002217368282
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -14.609168320105821,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": 45.06138392857143
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.7560763888889,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.93489583333334
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.7560763888889
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 5.505163239538234,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 1.7968749999999867
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 5.505163239538234
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 65.27742887769128,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 60.00000000000001,
        "score": 58.684593023255815
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 65.27742887769128
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 72.55934432838468,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 60.10584677419355
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 72.55934432838468
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 33.99933510638298,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 58.59375
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 33.99933510638298
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 59.23771512681159,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100,
        "score": 21.19140624999999
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 59.23771512681159
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 80.91397849462366,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 75
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 80.91397849462366
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 26.649305555555554,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 13.28125
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 26.649305555555554
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.816,
       "cc": 76.23375877839223,
       "cc_by_type": {
        "choice": 76.23375877839223
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 76.23375877839223
      },
      "noul": {
       "accuracy": 0.8133333333333334,
       "cc": 62.66666666666667,
       "cc_by_type": {
        "noul": 62.66666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 62.66666666666667
      },
      "score": {
       "accuracy": 0.648,
       "cc": 51.06646265794228,
       "cc_by_type": {
        "score": 51.06646265794228
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 51.06646265794228
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -14.646752450980395,
       "cc_by_type": {
        "choice": 36,
        "noul": -75,
        "score": -4.940257352941191
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.8333333333333334,
       "cc": 65.81281603360408,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 100,
        "score": 34.705528846153854
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 65.81281603360408
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 53.69452913279133,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100,
        "score": -22.65625
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 53.69452913279133
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": 9.276391934697855,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 33.33333333333333,
        "score": -23.6328125
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 9.276391934697855
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 32.395809744867144,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 73.9990234375
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 32.395809744867144
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 2.590654481132058,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": 100,
        "score": -154.49218750000006
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 2.590654481132058
      },
      "financial_crime": {
       "accuracy": 0.7333333333333333,
       "cc": 59.81640653056095,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 56.19991987179487
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 59.81640653056095
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 47.58298931980003,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 0,
        "score": 63.65489130434783
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 47.58298931980003
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 18.998842592592585,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 18.663194444444432
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.998842592592585
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 61.5794969569896,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 43.76775568181819
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 61.5794969569896
      },
      "lead_generation": {
       "accuracy": 0.64,
       "cc": 43.82631425691019,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 11.111111111111116,
        "score": 31.995738636363637
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 43.82631425691019
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -20.121212757616487,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": 2.001953125
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 71.39363736937514,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 60.00000000000001,
        "score": 77.21354166666666
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 71.39363736937514
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 82.56224264830222,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100,
        "score": 53.5308837890625
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 82.56224264830222
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 50.763020539309785,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 75.32169117647058
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 50.763020539309785
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 57.781549829774065,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 73.4375
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.781549829774065
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 48.64196217280016,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 50,
        "score": 25.482536764705888
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 48.64196217280016
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 40.69802989130435,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 46.73177083333333
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 40.69802989130435
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 97.50205592105264,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 92.5061677631579
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.50205592105264
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.53974842994678,
      84.88171158294799
     ],
     "I": [
      58.130438306540384,
      67.88277812548046
     ],
     "capability": [
      69.0012017096524,
      75.85146696574294
     ],
     "composite": {
      "A": [
       65.47342000127809,
       68.94694303660847
      ],
      "B": [
       63.90184880923016,
       68.72718774008524
      ],
      "C": [
       61.52814597282892,
       68.94694303660847
      ]
     },
     "seed": 68
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.91532974968945,
     "capability": 73.30803207834161,
     "intelligence": 63.700734406993774
    },
    "per_type": {
     "choice": {
      "accuracy": 0.816,
      "calibration": 85.16309130913092,
      "cc": 76.23375877839223,
      "cc_by_type": {
       "choice": 76.23375877839223
      },
      "cc_tiered": 77.09500869987218,
      "cc_tiered_open": 80.09787615590605,
      "cc_tiered_sealed": 74.0921412438383,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.23375877839223
     },
     "noul": {
      "accuracy": 0.8133333333333334,
      "calibration": 81.4622654155496,
      "cc": 62.66666666666667,
      "cc_by_type": {
       "noul": 62.66666666666667
      },
      "cc_tiered": 61.98535225506068,
      "cc_tiered_open": 63.46997929606625,
      "cc_tiered_sealed": 60.500725214055116,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 62.66666666666667
     },
     "score": {
      "accuracy": 0.648,
      "calibration": 82.12063252438782,
      "cc": 51.06646265794228,
      "cc_by_type": {
       "score": 51.06646265794228
      },
      "cc_tiered": 52.02184226604846,
      "cc_tiered_open": 56.830186130371516,
      "cc_tiered_sealed": 47.2134984017254,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 51.06646265794228
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.026666666666666616,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 63.79558015456306,
    "Emphasis on Accuracy 60:20:20": 63.757607983292516,
    "Emphasis on Cost 20:20:60": 61.995746479322406,
    "Emphasis on Speed 20:60:20": 65.74336798715605,
    "Intelligence only": 63.70073440699377
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "ScaleDown",
   "axes": {
    "calibration": 82.82809086203193,
    "cost": 50.78732378203712,
    "intelligence": 21.787775467356628,
    "speed": 83.72843286582474
   },
   "base_model": null,
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0328225927768466,
      "ece_score": 93.43548144463068,
      "mean_tvd": null,
      "n": 749,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 92.56341695114746,
      "score_before_time_limit": 93.43548144463068,
      "time_limit_failed_share": 0.009333333333333334
     },
     "noul": {
      "abstention_rate": 0.9064171122994652,
      "brier": 0.21132460936574093,
      "ece": 0.15882660638443544,
      "n": 374,
      "score": 67.87076043658963,
      "score_before_time_limit": 68.2346787231129,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.9839142091152815,
      "interval90_width": 0.7225201072386056,
      "n": 373,
      "nrps": 0.1191379439544878,
      "score": 88.05009519835869,
      "score_before_time_limit": 89.23996134968786,
      "time_limit_failed_share": 0.013333333333333334,
      "top_ece": 0.0480314145258774
     }
    },
    "score": 82.82809086203193
   },
   "capability": 52.307933164694276,
   "code_revision": "Vendor-hosted API: no vendor code ran locally; weights and serving code are not public. First-party adapter scaledown_classify_v16.py (sha256 f78d1f8251facaa60d10e9b5c5033dac206024bee5469968706d3ec2f61b8680) + spec scaledown-classify-1.json (sha256 47032611bea155be78f8acf8c2e41d73885e5e48829e79662d00fcc38cc64ec7); independent review PASS_WITH_MUSTS, musts fixed before dispatch.",
   "composite_ci95": {
    "A": [
     4.672849375867131,
     11.765050469662265
    ],
    "B": [
     3.7201929696864933,
     9.959843262217815
    ],
    "C": [
     3.245034288796619,
     8.170173937265462
    ]
   },
   "cost": {
    "basis": "ScaleDown vendor list price, USD 0.05/M input tokens; output is free. Explicit vendor reference scaledown:classify-1 (vendor list price), with no base-model price. The frozen pricing module has no key for this API, so this row uses an explicit reference at that price. The scored cost includes the frozen scorer's usage fallback for four HTTP 504 rows: USD 0.04369407407407409 per 1,000 decisions on the common 1,485-item basis (0.67643x the USD 0.06459465517241379 cap); USD 0.082679 per 1,000 over all 1,500. Provider-reported usage alone gives USD 0.035613266 per 1,000 common items and USD 0.074679 over all items, excluding usage estimates for those failures. Cost kind mixed: four rows have estimated usage. Not a bill or quote.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.082679
    },
    "kind": "mixed",
    "usage_estimated_rows": 4,
    "usd_per_1000": 0.04369407407407409
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 14,
    "n_over_limit_answered_ok": 10,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "STOCK PROMPT: the canonical question is sent unchanged in ScaleDown's native Jev-compatible format (POST /v1/scaledown, model classify-1). The vendor asked for prompt tuning with its SLM_Agent method first; that method was applied by us on 40 public dev items only (no vendor code executed), and no variant beat the run-to-run noise floor, so the stock prompt is the only measured configuration. 4 items ended in HTTP 504 (gateway timeout) and are scored wrong; no retry. Free trial tokens plus a USD 5 evaluator-paid credit covered the run.",
   "display": "classify-1 (ScaleDown)",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-24",
    "release": "v1.7.3",
    "results_sha256": "31efd92d925131d31de1656d55cadaefcf858ee776100eb50e35932ac5b354e9"
   },
   "endpoint_condition": "Vendor-hosted api.scaledown.xyz/v1/scaledown API, model classify-1 (served model checked on every row; all 1,500 rows named classify-1), through first-party adapter scaledown_classify_v16; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py on 24 Oct 2026; the full-set exposure gate logged all 1,200 sealed items to provider scaledown before the first request. Native probabilities on all three question types; string states sent unchanged.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 23.494782199971077,
    "I_sealed": 20.08076873474218,
    "base": 21.787775467356628,
    "excess": -6.425513305126813,
    "gap": 3.414013465228898,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 44.97100315222714,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 32.05410506448569,
       "judge": 52.537903757415954,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -100,
       "judge": -80,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 25.5133434476861,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.7781499849444,
       "hard": 19.54744192589989,
       "judge": 35.38519255206841,
       "standard": 4.504969566055905
      }
     },
     "sealed|choice": {
      "cc": 42.194486380067914,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 48.538754764930125,
       "hard": 32.86689025095408,
       "judge": 41.82174338883449,
       "standard": 58.23665893271462
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -75.86206896551724,
       "hard": -90.9090909090909,
       "judge": -93.84615384615384,
       "standard": -64.17910447761194
      }
     },
     "sealed|score": {
      "cc": 18.047819824158623,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 35.58892918028347,
       "hard": 10.509426612841189,
       "judge": 17.760290836455994,
       "standard": 24.785345050285013
      }
     }
    }
   },
   "jevbench_score": 8.476551950321573,
   "key": "scaledown-classify-1",
   "last_measured_on": "2026-10-24",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.7.3",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "classify-1 (vendor-hosted API; weights not public)",
   "new_in": "v1.7.3",
   "noul_decisive": {
    "acc_among_decisive": 1,
    "decisive": 35,
    "decisive_rate": 0.0938337801608579,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 220,
   "ranked": true,
   "ranks": {
    "A": 220,
    "B": 221,
    "C": 221,
    "capability": 205
   },
   "repo": "https://scaledown.ai",
   "scores": {
    "A": 8.476551950321573,
    "B": 7.006689864970686,
    "C": 5.886494409945535
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5046778135074419,
    "p50_s_raw": 0.5046778135074419,
    "p95_s_adjusted": 0.8397354750937666,
    "p95_s_raw": 0.8397354750937666
   },
   "status": {
    "answered_ok": 1496,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5909090909090909,
       "cc": -10.666206189926086,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": -50,
        "score": -48.66528523644493
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.33663366336633666,
       "cc": -17.53828239080254,
       "cc_by_type": {
        "choice": 36.37327677624603,
        "noul": -100,
        "score": 11.011876051346349
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.3465346534653465,
       "cc": -23.057397205506685,
       "cc_by_type": {
        "choice": 38.211382113821145,
        "noul": -100,
        "score": -7.3835737303411975
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.3958333333333333,
       "cc": 2.547459949373632,
       "cc_by_type": {
        "choice": 71.42857142857143,
        "noul": -100,
        "score": 36.213808419549466
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 2.547459949373632
      },
      "jaggedness": {
       "accuracy": 0.3277777777777778,
       "cc": -13.33111523931853,
       "cc_by_type": {
        "choice": 27.81520692974015,
        "noul": -83.78378378378379,
        "score": 15.975231136088052
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.49201277955271566,
       "cc": 16.58069562448333,
       "cc_by_type": {
        "choice": 48.48991811217751,
        "noul": -36.08247422680413,
        "score": 37.3346429880766
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 16.58069562448333
      },
      "long_state": {
       "accuracy": 0.2,
       "cc": -43.417982168029745,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100,
        "score": -27.028140052476335
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -31.105400706660635,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -100,
        "score": 5.652870044966551
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.4,
       "cc": -8.401030900371623,
       "cc_by_type": {
        "choice": 44.24519074013694,
        "noul": -100,
        "score": 30.551716558748186
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.3372093023255814,
       "cc": -29.365420042260922,
       "cc_by_type": {
        "choice": 43.33333333333332,
        "noul": -100,
        "score": -31.429593460116088
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.6666666666666666,
       "cc": 57.3901464713715,
       "cc_by_type": {
        "choice": 57.3901464713715
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 57.3901464713715
      },
      "perturb_pair": {
       "accuracy": 0.484375,
       "cc": -1.007614951330936,
       "cc_by_type": {
        "choice": 39.93993993993994,
        "noul": -100,
        "score": 57.03721520606726
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.42272727272727273,
       "cc": -9.73751981494302,
       "cc_by_type": {
        "choice": 52.205882352941195,
        "noul": -100,
        "score": 18.581558202229743
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.1875,
       "cc": -45.937517302071406,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": -100,
        "score": -23.52683762049994
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.2857142857142857,
       "cc": -42.10526315789474,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.46153846153846156,
       "cc": 15.03271897151216,
       "cc_by_type": {
        "choice": 37.78801843317972,
        "noul": -33.333333333333336,
        "score": 40.6434718146901
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 15.03271897151216
      },
      "de": {
       "accuracy": 0.5,
       "cc": 9.426749170259418,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": -75,
        "score": 51.06785813024728
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 9.426749170259418
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 5.387342151544179,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -100,
        "score": 83.20113259988393
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 5.387342151544179
      },
      "en": {
       "accuracy": 0.4126846220677672,
       "cc": -6.199340123447555,
       "cc_by_type": {
        "choice": 44.2037563612843,
        "noul": -81.29496402877699,
        "score": 18.493187297150026
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.5483870967741935,
       "cc": 8.100430998083267,
       "cc_by_type": {
        "choice": 60.848287112561174,
        "noul": -25,
        "score": -11.546994118311371
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 8.100430998083267
      },
      "fi": {
       "accuracy": 0,
       "cc": -22.38593627582843,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100,
        "score": 63.98973215612127
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.42857142857142855,
       "cc": -8.361590469741543,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": -100,
        "score": 38.444640355481255
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.3333333333333333,
       "cc": -24.140646616230526,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100,
        "score": -22.142610239753036
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.6666666666666666,
       "cc": 5.863255649147139,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": -100,
        "score": 31.619385030060286
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 5.863255649147139
      },
      "ja": {
       "accuracy": 0.16,
       "cc": -25.91213603285095,
       "cc_by_type": {
        "choice": -14.376489277204131,
        "noul": -80,
        "score": 16.64008117865129
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -17.894184345779824,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": 35.20633585154942
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.6428571428571429,
       "cc": 23.52372643283176,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 70.57117929849528
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 23.52372643283176
      },
      "nl": {
       "accuracy": 0.3,
       "cc": -30.464207419154246,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": -39.44457030941078
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.46153846153846156,
       "cc": 5.647599423614661,
       "cc_by_type": {
        "choice": 54.29538721963606,
        "noul": -60,
        "score": 22.64741105120792
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 5.647599423614661
      },
      "pt": {
       "accuracy": 0.6428571428571429,
       "cc": 15.761110940502286,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": -60,
        "score": 49.71114661054635
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 15.761110940502286
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": -7.918198558887582,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": -100,
        "score": 27.30923411057131
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -39.024340860461855,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100,
        "score": -51.85563127703776
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 3.307902839273879,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": 74.4398375500797
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 3.307902839273879
      },
      "zh": {
       "accuracy": 0.3,
       "cc": -30.87603744758397,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100,
        "score": -16.35692590207396
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.5653333333333334,
       "cc": 43.856560592433844,
       "cc_by_type": {
        "choice": 43.856560592433844
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 43.856560592433844
      },
      "noul": {
       "accuracy": 0.09333333333333334,
       "cc": -81.33333333333333,
       "cc_by_type": {
        "noul": -81.33333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.4266666666666667,
       "cc": 20.767057167986668,
       "cc_by_type": {
        "score": 20.767057167986668
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 20.767057167986668
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -34.66351608264553,
       "cc_by_type": {
        "choice": 20,
        "noul": -100,
        "score": -23.990548247936584
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -28.909740728640543,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -100,
        "score": -12.195060695238391
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 4.559150971409042,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": -100,
        "score": 62.457940719105174
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 4.559150971409042
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -32.121284533711,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -100,
        "score": 8.89930429360386
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": -1.5302353780446747,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -100,
        "score": 38.887554735431195
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -78.33824039809011,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100,
        "score": -146.96440672886152
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": -5.263394470953986,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -100,
        "score": 23.425502861647864
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": -5.73626261283147,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100,
        "score": 35.05602052387492
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.2,
       "cc": -10.506258968302104,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -100,
        "score": 30.98122309509369
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -32.60737369559937,
       "cc_by_type": {
        "choice": 22.912205567451824,
        "noul": -100,
        "score": -20.734326654249923
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.28,
       "cc": -19.150680306312765,
       "cc_by_type": {
        "choice": 41.860465116279066,
        "noul": -100,
        "score": 0.6874939647826372
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -53.935848404475244,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -32.775287148909584
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5185185185185185,
       "cc": -2.038133746750438,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 16.91822831828993
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.5714285714285714,
       "cc": 29.51514079030297,
       "cc_by_type": {
        "choice": 61.42857142857144,
        "noul": -9.090909090909093,
        "score": 36.20776003324657
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 29.51514079030297
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 7.42457875054303,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 56.822680589441
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 7.42457875054303
      },
      "recruiting": {
       "accuracy": 0.5555555555555556,
       "cc": -4.107592505024665,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -100,
        "score": 21.103406328937147
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -13.011365405747886,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -100,
        "score": 20.07920427536718
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.391304347826087,
       "cc": -13.243361938976696,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100,
        "score": 18.240928675823533
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.7083333333333334,
       "cc": 20.349313001199747,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": -100,
        "score": 82.40025217085902
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 20.349313001199747
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      76.65403551196366,
      83.9162808157726
     ],
     "I": [
      17.266753746925517,
      24.895368297491746
     ],
     "capability": [
      47.686868271506455,
      53.589923044650135
     ],
     "composite": {
      "A": [
       4.672849375867131,
       11.765050469662265
      ],
      "B": [
       3.7201929696864933,
       9.959843262217815
      ],
      "C": [
       3.245034288796619,
       8.170173937265462
      ]
     },
     "seed": 69
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.82809086203193,
     "capability": 52.307933164694276,
     "intelligence": 21.787775467356628
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5653333333333334,
      "calibration": 92.56341695114746,
      "cc": 43.856560592433844,
      "cc_by_type": {
       "choice": 43.856560592433844
      },
      "cc_tiered": 43.58274476614753,
      "cc_tiered_open": 44.97100315222714,
      "cc_tiered_sealed": 42.194486380067914,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 43.856560592433844
     },
     "noul": {
      "accuracy": 0.09333333333333334,
      "calibration": 67.87076043658963,
      "cc": -81.33333333333333,
      "cc_by_type": {
       "noul": -81.33333333333333
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.4266666666666667,
      "calibration": 88.05009519835869,
      "cc": 20.767057167986668,
      "cc_by_type": {
       "score": 20.767057167986668
      },
      "cc_tiered": 21.78058163592236,
      "cc_tiered_open": 25.5133434476861,
      "cc_tiered_sealed": 18.047819824158623,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 20.767057167986668
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.009333333333333305,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 7.347401123964962,
    "Emphasis on Accuracy 60:20:20": 5.607046211426425,
    "Emphasis on Cost 20:20:60": 8.120862405107507,
    "Emphasis on Speed 20:60:20": 9.361456533393355,
    "Intelligence only": 4.137125204326576
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ironbcc (Hugging Face)",
   "axes": {
    "calibration": 87.96306938738428,
    "cost": 68.563136114908,
    "intelligence": 38.312942577836566,
    "speed": 90.67341977526846
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.058193423604320185,
      "ece_score": 88.36131527913597,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 88.36131527913597,
      "score_before_time_limit": 88.36131527913597,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.42627345844504017,
      "brier": 0.12368613516681048,
      "ece": 0.06544694571318402,
      "n": 373,
      "score": 86.9106108573632,
      "score_before_time_limit": 86.9106108573632,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9435483870967742,
      "interval90_width": 0.4249551971326164,
      "n": 372,
      "nrps": 0.09008988865783839,
      "score": 88.61728202565371,
      "score_before_time_limit": 88.61728202565371,
      "time_limit_failed_share": 0,
      "top_ece": 0.06878223541454365
     }
    },
    "score": 87.96306938738428
   },
   "capability": 63.13800598261042,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 748,494 input + 1,485 output tokens over 1,485 decisions = 505.036363636 tokens/decision. GPU-time cost USD 0.0111659444476694/1,000; old list-price cost USD 0.0505536363636364/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0111659444476694/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0500481
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011165944447669357
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Seb-9B (ironbcc, Qwen3.5-9B fine-tune, one forward pass, stock vLLM, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-20",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x NVIDIA A100-SXM4 80 GB (FIN-02, Finland, EU, pay-as-you-go non-spot, USD 2.148/h). Stock vLLM 0.30.0 (digest-pinned) + our shim in containers with --network none (cut proven from inside the measuring container before the sealed input), Hub offline, credentials unset, all 20 repo files sha256-verified; driven by the unchanged pinned evaluator driver, one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 20 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 44.92534796116673,
    "I_sealed": 31.700537194506396,
    "base": 38.312942577836566,
    "excess": 3.385283996304622,
    "gap": 13.224810766660333,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 69.07949695507523,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 62.25228059138094,
       "judge": 76.26895187870798,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 17.61421670117322,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 13.043478260869556,
       "judge": -9.999999999999998,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 48.082330227251745,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.774986302195465,
       "hard": 46.6348940947645,
       "judge": 48.630522457567935,
       "standard": 42.80858610928006
      }
     },
     "sealed|choice": {
      "cc": 57.06524108576487,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 77.12833545108006,
       "hard": 51.081299872788236,
       "judge": 55.04407443682666,
       "standard": 62.033326302467835
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 37.93103448275863,
       "hard": -14.54545454545455,
       "judge": -10.769230769230765,
       "standard": 16.417910447761198
      }
     },
     "sealed|score": {
      "cc": 38.03637049775433,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 67.25874927356028,
       "hard": 29.443486400788153,
       "judge": 33.84604716898835,
       "standard": 46.896434296932675
      }
     }
    }
   },
   "jevbench_score": 37.2305204893958,
   "key": "seb-9b",
   "last_measured_on": "2026-10-20",
   "licence": "Apache-2.0 (LICENSE in the repo and HF metadata; base Qwen/Qwen3.5-9B Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.6.35",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "ironbcc/seb-9b@4ae564d62fb69a86d5624c9176c6e271c539485f",
   "noul_decisive": {
    "acc_among_decisive": 0.9392523364485982,
    "decisive": 214,
    "decisive_rate": 0.5737265415549598,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.96306938738428,
     "cost": 48.88742799396291,
     "intelligence": 38.312942577836566,
     "speed": 90.8001817381692
    },
    "board_release": "v1.6.43",
    "capability": 63.13800598261042,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 45,
    "capability_rank": 70,
    "composite_ci95": {
     "A": [
      22.635579367850564,
      41.68054108791833
     ],
     "B": [
      20.014613889486654,
      38.49504774745768
     ],
     "C": [
      15.719152338785115,
      28.944820199943283
     ]
    },
    "jevbench_score": 32.56935976147238,
    "measured_in": "v1.6.35",
    "method": "v1.6",
    "open_board_rank": 65,
    "rank": 95,
    "ranks": {
     "A": 95,
     "B": 97,
     "C": 97,
     "capability": 112
    },
    "scores": {
     "A": 32.56935976147238,
     "B": 29.53085969501141,
     "C": 22.617610945466925
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24929025000008095,
     "p50_s_raw": 0.04964512500004048,
     "p95_s_adjusted": 0.3336387865998859,
     "p95_s_raw": 0.09181939329994293
    },
    "views": {
     "Balanced 33:33:33": 29.250716419100634,
     "Emphasis on Accuracy 60:20:20": 25.56750030507671,
     "Emphasis on Cost 20:20:60": 28.498993943725626,
     "Emphasis on Speed 20:60:20": 35.26033540949847,
     "Intelligence only": 21.50556650674145
    }
   },
   "rank": 153,
   "ranked": true,
   "ranks": {
    "A": 153,
    "B": 154,
    "C": 154,
    "capability": 143
   },
   "repo": "https://huggingface.co/ironbcc/seb-9b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 37.2305204893958,
    "B": 32.91813021766584,
    "C": 25.854528117635972
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2500642110000172,
    "p50_s_raw": 0.050032105500008583,
    "p95_s_adjusted": 0.3424573425000972,
    "p95_s_raw": 0.0962286712500486
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "supersedes_base_row": "v1.6.1",
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "No author code exists in the repository (security scan PASS: no .py, no auto_map, no pickle; config, tokenizer, chat template and preprocessor files carry the same git oids as Qwen/Qwen3.5-9B main). Weights at 4ae564d62fb69a86d5624c9176c6e271c539485f are byte-identical to the 7 Oct measured revision ac57b36b (all four shard LFS sha256 equal; only README.md changed and assets/benchmarks-20261019.json added). Served by the stock vLLM 0.30.0 OpenAI server (image pinned by digest vllm/vllm-openai:v0.30.0@sha256:8a69ffad...) with the author's documented command; the only change from the 7 Oct run is --max-model-len 32768 (was 8192), as the author's current card states. The /v1/systemone wire layer is OUR reviewed shim wire adapter (sha256 e617f911...), unchanged since 7 Oct: it builds the author's documented chat request and renormalises exp(top_logprobs) over the option labels; it applies no temperature and no rule of its own. RE-MEASUREMENT at the author's request (GitHub issue #222): the earlier row (v1.6.1 pool, 7 Oct) lost 23 items to the card's then --max-model-len 8192 and was listed unpriced because no base model was declared. AUTHOR CONTEXT CAP: at the author's current 32,768-token limit vLLM refuses the 15 items of this draw that are 76k-81k tokens long (HTTP 400, mapped to a per-question refusal by our shim); they count as wrong, like d3-turbo and d1-ultra on this draw. OWN HTTP SHIM: the wire layer is ours (see the serving description). TRAINING DATA, the author's own statements (unverified): the card says training included the typed-decisions benchmark's own training split (not its test cases) and that about 45 candidates were compared on the author's own sealed suite before release (selection optimism); issue #222 states that JevBench public items were not used in training or development. All card numbers are self-reported. HARDWARE: measured on an A100 80 GB (the RTX PRO 6000 was out of stock); latency reflects that card. One 85-token warm-up request preceded the scored loop and is not part of any scored row. This re-measurement supersedes the 7 Oct v1.6.1 measurement of the same system on live boards; the frozen v1.6.1 archive retains that row unchanged. The author has declared base Qwen/Qwen3.5-9B since card revision 456cc58; the weights are unchanged from the 7 Oct measurement. SUPERSEDED BY: seb-9b-v2 (/jev-models/seb-9b-v2); the predecessor measurement remains on this board.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 45.223746707381544,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": -30.995426544522033
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 45.223746707381544
      },
      "calibration_gen": {
       "accuracy": 0.4158415841584158,
       "cc": 3.6716931899495946,
       "cc_by_type": {
        "choice": 23.647932131495228,
        "noul": -19.999999999999996,
        "score": 7.36714743835355
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 3.6716931899495946
      },
      "composite_gen": {
       "accuracy": 0.6831683168316832,
       "cc": 39.202518638774364,
       "cc_by_type": {
        "choice": 70.73170731707317,
        "noul": -11.111111111111116,
        "score": 57.98695971036105
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 39.202518638774364
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 77.84537293234432,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 73.53611879703296
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 77.84537293234432
      },
      "jaggedness": {
       "accuracy": 0.40555555555555556,
       "cc": 4.866912866461033,
       "cc_by_type": {
        "choice": 26.37151106833495,
        "noul": -45.945945945945944,
        "score": 34.17517347699409
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 4.866912866461033
      },
      "llm_router": {
       "accuracy": 0.792332268370607,
       "cc": 63.347262987355755,
       "cc_by_type": {
        "choice": 77.01857885004843,
        "noul": 52.57731958762886,
        "score": 60.44589052439
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 63.347262987355755
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -14.960395410016568,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75,
        "score": 4.345617893661635
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6327272727272727,
       "cc": 34.93177969967219,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": -8.333333333333337,
        "score": 47.36408162700266
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 34.93177969967219
      },
      "negation_pair_gen": {
       "accuracy": 0.5930232558139535,
       "cc": 19.717304053044884,
       "cc_by_type": {
        "choice": 76.66666666666666,
        "noul": -33.333333333333336,
        "score": 15.818578825801332
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 19.717304053044884
      },
      "option_order": {
       "accuracy": 0.7291666666666666,
       "cc": 65.37949400798935,
       "cc_by_type": {
        "choice": 65.37949400798935
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 65.37949400798935
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 74.42814201458951,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 80,
        "score": 70.31145307079557
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 74.42814201458951
      },
      "usecase": {
       "accuracy": 0.5772727272727273,
       "cc": 35.62797906666504,
       "cc_by_type": {
        "choice": 58.33333333333336,
        "noul": 8.333333333333325,
        "score": 40.21727053332845
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 35.62797906666504
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -0.9727306051689837,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 0,
        "score": -45.775334672649805
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 50.649429931030646,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 33.33333333333333,
        "score": 60.089610837638816
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 50.649429931030646
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 68.05211316394812,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 53.05014480157888
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 68.05211316394812
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 27.624535047530582,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 99.81960141819697
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 27.624535047530582
      },
      "en": {
       "accuracy": 0.6211989574283232,
       "cc": 35.82151142994442,
       "cc_by_type": {
        "choice": 58.48232264678242,
        "noul": 7.913669064748197,
        "score": 41.06854257830265
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 35.82151142994442
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 67.2468941424042,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 21.316538870932057
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 67.2468941424042
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 18.50070873628631,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -33.333333333333336,
        "score": 54.409230033995534
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 18.50070873628631
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 73.950788792126,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 85.38177814108387
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 73.950788792126
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 30.07009803937503,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25,
        "score": 48.72984719075079
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 30.07009803937503
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 79.07764418870396,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 51.26255064873071
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 79.07764418870396
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 28.371926564196414,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 0,
        "score": 28.0069631715408
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 28.371926564196414
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 13.914615064090782,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 41.743845192272346
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 13.914615064090782
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.78145276512208,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 97.0110249620329
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.78145276512208
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 9.24592290572977,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 53.71179469121527
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.24592290572977
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 33.74777631633108,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 36.4781998757812
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 33.74777631633108
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 69.11348793153977,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 49.76827758365878
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 69.11348793153977
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 5.5967829438067485,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 53.3860935122713
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 5.5967829438067485
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 5.8269113862106705,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0,
        "score": -60.78013540658538
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 5.8269113862106705
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 70.4456198595988,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 75.8529886110545
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 70.4456198595988
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 7.847467533423753,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -43.1242640663954
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 7.847467533423753
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.692,
       "cc": 60.21737882470005,
       "cc_by_type": {
        "choice": 60.21737882470005
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 60.21737882470005
      },
      "noul": {
       "accuracy": 0.536,
       "cc": 7.200000000000006,
       "cc_by_type": {
        "noul": 7.200000000000006
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 7.200000000000006
      },
      "score": {
       "accuracy": 0.5893333333333334,
       "cc": 41.5783099741794,
       "cc_by_type": {
        "score": 41.5783099741794
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 41.5783099741794
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -11.399722591362355,
       "cc_by_type": {
        "choice": 36,
        "noul": -75,
        "score": 4.800832225912933
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 13.327689404010428,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": 3.406118591603402
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 13.327689404010428
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 52.77491020433876,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50,
        "score": 73.36538101952034
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 52.77491020433876
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -32.8006986415953,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -16.53075089554614
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 27.266093112388443,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 44.11711991687546
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 27.266093112388443
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -52.26868973955401,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": 0,
        "score": -168.75575475325323
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 21.869899893472432,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -42.85714285714286,
        "score": 37.878607243442495
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 21.869899893472432
      },
      "gaming": {
       "accuracy": 0.22727272727272727,
       "cc": -17.58128304147245,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100,
        "score": 30.87984425537359
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 18.174596396111358,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 77.02378918833408
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 18.174596396111358
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 26.355259817736492,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 0,
        "score": 30.457642408241593
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 26.355259817736492
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 46.59774716170659,
       "cc_by_type": {
        "choice": 100,
        "noul": 11.111111111111116,
        "score": 28.682130374008665
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 46.59774716170659
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -41.51961222180168,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": -62.19324526755556
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5925925925925926,
       "cc": 31.68684660421144,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -19.999999999999996,
        "score": 38.09316937117556
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 31.68684660421144
      },
      "model_routing": {
       "accuracy": 0.8341013824884793,
       "cc": 68.73172018279341,
       "cc_by_type": {
        "choice": 84.80519480519482,
        "noul": 72.72727272727273,
        "score": 48.66269301591265
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 68.73172018279341
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 3.1159072574523257,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 55.4129808894395
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 3.1159072574523257
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 58.39257887745197,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 58.557495221028155
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 58.39257887745197
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 12.422368601973668,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50,
        "score": 16.823756052226415
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 12.422368601973668
      },
      "scientific_discovery": {
       "accuracy": 0.5217391304347826,
       "cc": 10.835094112201524,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -66.66666666666667,
        "score": 42.65020987283646
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 10.835094112201524
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.60108803048422,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.80326409145266
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.60108803048422
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.3070763387534,
      89.00056245181912
     ],
     "I": [
      32.95519131864929,
      42.587296967160135
     ],
     "capability": [
      58.747434360337614,
      65.161057795857
     ],
     "composite": null,
     "seed": 69
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.96306938738428,
     "capability": 63.13800598261042,
     "intelligence": 38.312942577836566
    },
    "per_type": {
     "choice": {
      "accuracy": 0.692,
      "calibration": 88.36131527913597,
      "cc": 60.21737882470005,
      "cc_by_type": {
       "choice": 60.21737882470005
      },
      "cc_tiered": 63.07236902042005,
      "cc_tiered_open": 69.07949695507523,
      "cc_tiered_sealed": 57.06524108576487,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.21737882470005
     },
     "noul": {
      "accuracy": 0.536,
      "calibration": 86.9106108573632,
      "cc": 7.200000000000006,
      "cc_by_type": {
       "noul": 7.200000000000006
      },
      "cc_tiered": 8.80710835058661,
      "cc_tiered_open": 17.61421670117322,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 7.200000000000006
     },
     "score": {
      "accuracy": 0.5893333333333334,
      "calibration": 88.61728202565371,
      "cc": 41.5783099741794,
      "cc_by_type": {
       "score": 41.5783099741794
      },
      "cc_tiered": 43.05935036250304,
      "cc_tiered_open": 48.082330227251745,
      "cc_tiered_sealed": 38.03637049775433,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 41.5783099741794
     }
    },
    "raw_on_A": {
     "calibration": 84.71000182672668,
     "capability": 62.694073356703875,
     "intelligence": 40.67814488668107
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 34.06118077990313,
    "Emphasis on Accuracy 60:20:20": 28.2512533387939,
    "Emphasis on Cost 20:20:60": 36.29564471946234,
    "Emphasis on Speed 20:60:20": 39.79524396080314,
    "Intelligence only": 22.49554490524478
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ironbcc (Hugging Face)",
   "axes": {
    "calibration": 83.5320529378626,
    "cost": 68.563136114908,
    "intelligence": 46.11278573865326,
    "speed": 91.9752040844015
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09739173773645653,
      "ece_score": 80.5216524527087,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.5216524527087,
      "score_before_time_limit": 80.5216524527087,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.30294906166219837,
      "brier": 0.13463719291769388,
      "ece": 0.0866352733474072,
      "n": 373,
      "score": 82.67294533051856,
      "score_before_time_limit": 82.67294533051856,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9354838709677419,
      "interval90_width": 0.3573028673835125,
      "n": 372,
      "nrps": 0.09039978855699243,
      "score": 87.40156103036057,
      "score_before_time_limit": 87.40156103036057,
      "time_limit_failed_share": 0,
      "top_ece": 0.08078449541789799
     }
    },
    "score": 83.5320529378626
   },
   "capability": 64.82241933825793,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 9B active parameters; throughput 27344.4444444 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 748,494 input + 1,485 output tokens over 1,485 decisions = 505.036363636 tokens/decision. GPU-time cost USD 0.0111659444476694/1,000; old list-price cost USD 0.0505536363636364/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0111659444476694/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0500481
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011165944447669357
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Seb-9B-v2 (ironbcc, Qwen3.5-9B fine-tune, one forward pass, stock vLLM, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.574/h. Stock vLLM 0.30.0 (127.0.0.1:8000, digest-pinned image) and our shim (the /v1/systemone endpoint) run in containers with network none, driven one request at a time by the unchanged pinned pod_driver_v16. The standard self-hosted x2 + 0.15 s latency adjustment applies. Network cut proven from inside the container before any item hash or model load (loopback only, no default route, DNS and TCP fail); read-only code, read-only model and input, empty credential variables (names-only proof), HF offline flags. Measured 23 Oct 2026 (VM 07:12-07:25Z; measurement wrapper 250 s; scored loop 63.3 s per the receipt) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, re-checked inside the container). The input was shredded from the node and its absence verified, and the VM was removed by exact id afterwards (teardown confirmed 07:25Z).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 53.38140037125771,
    "I_sealed": 38.844171106048805,
    "base": 46.11278573865326,
    "excess": 4.697702494853196,
    "gap": 14.537229265208907,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 71.25353822050559,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 64.1396665618119,
       "judge": 68.35860250494397,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 38.33471359558317,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 50.55594929768438,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.48140097601048,
       "hard": 47.61342904269908,
       "judge": 45.68452143173856,
       "standard": 55.28540576741062
      }
     },
     "sealed|choice": {
      "cc": 59.787341513336614,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 71.41041931385007,
       "hard": 53.16294668671215,
       "judge": 59.4515181194907,
       "standard": 67.72832735709765
      }
     },
     "sealed|noul": {
      "cc": 16.844494671566217,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 48.27586206896552,
       "hard": 10.909090909090914,
       "judge": 4.615384615384621,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 39.90067713324359,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 69.80403626056912,
       "hard": 29.99445855403019,
       "judge": 36.563140254082136,
       "standard": 49.76774004674978
      }
     }
    }
   },
   "jevbench_score": 57.552603401286994,
   "key": "seb-9b-v2",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 (LICENSE in the repo and HF metadata; base Qwen/Qwen3.5-9B Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "ironbcc/Seb-9B-v2@ca903824a11dd763a8e0dec5bce5e13e3e4166db",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9076923076923077,
    "decisive": 260,
    "decisive_rate": 0.6970509383378016,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 114,
   "ranked": true,
   "ranks": {
    "A": 114,
    "B": 116,
    "C": 119,
    "capability": 132
   },
   "repo": "https://huggingface.co/ironbcc/Seb-9B-v2",
   "scores": {
    "A": 57.552603401286994,
    "B": 52.632763359177375,
    "C": 39.9670856953382
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22097171999998863,
    "p50_s_raw": 0.03548585999999432,
    "p95_s_adjusted": 0.2871725307000929,
    "p95_s_raw": 0.06858626535004646
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "No author code runs at all (static review PASS: the repo's four serve/*.py files are excluded from the manifest and never copied to the node; the run fails if any .py/.sh/.so/.pyc/.pkl/.pt/.bin exists in /model). Weights: ironbcc/Seb-9B-v2 at ca903824a11dd763a8e0dec5bce5e13e3e4166db (Apache-2.0, public, not gated; Qwen/Qwen3.5-9B plus a merged LoRA, scale 20, 248 modules; 14 manifest files, 4 bf16 shards = 19,306,311,000 bytes; manifest sha256 d138599972ed6caa640987b3bd175a24e8f0d76e4485317d1de6240044d660d4; every file sha256-verified against the Hub oids). The Hub HEAD later moved to c01ef79eb54b491c83ed1260ab8444cbc662e65c by README, blog and media commits only: the 19 weight, config and tokenizer oids are identical at both commits, and the run loads exactly the pinned commit. config.json, chat_template.jinja, tokenizer.json, tokenizer_config.json and the preprocessor files carry the same git oids as the v1 row seb-9b; vocab.json and merges.txt on the Hub are truncated at 1,048,576 bytes (unused by the fast tokenizer; equivalence with v1 proven offline on 467 texts and by the usage match below). Served by the stock vLLM 0.30.0 OpenAI server (image pinned by digest vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90, no derived image) with the author's published command (vllm serve /model --served-model-name seb-9b-v2 --logprobs-mode processed_logprobs --max-logprobs 32 --generation-config vllm --max-model-len 32768 --dtype bfloat16 --max-num-seqs 128 --enable-prefix-caching --gpu-memory-utilization 0.85 --host 127.0.0.1 --port 8000); the only delta from the v2 card is --gpu-memory-utilization 0.85, kept from the v1 recipe (it only sizes the KV cache). vllm 0.30.0, torch 2.13.0+cu130, transformers 5.17.0, tokenizers 0.23.2, flashinfer 0.6.18.post1. The /v1/systemone wire layer is OUR reviewed shim seb_systemone_r56.py (sha256 e617f911e9218359b35eb8d7761ae6674c75631398efdd76996b1741e6a17b99, byte-identical to the v1 row's): it builds the author's documented chat request (max_tokens 1, temperature 0, top_logprobs, structured_outputs.choice, thinking off) and renormalises exp(top_logprobs) over the option labels; it applies no temperature and no rule of its own. The runner was reviewed and findings resolved in the preregistration amendments before the run. FIRST MEASUREMENT OF THIS VERSION. NEW VERSION OF AN EXISTING MODEL: Seb-9B-v2 has new weights (all four shards differ from ironbcc/seb-9b); the v1 row seb-9b measured 20 Oct 2026 stays as it is. The submission reference is not recorded in the files read for this entry. The two serving versions are reported separately. SECOND USE OF THE DRAW FOR THE FAMILY: evaluator exposure and device ruling admitted this as a same-author second use (v1 consumed the frozen draw on 20 Oct, on file; the v2 weights were published after that score), with disclosure; any further ironbcc row goes to the next draw. AUTHOR CONTEXT CAP: at the card's 32,768-token limit vLLM refuses the 15 items of this draw that are 76,344-80,504 tokens long (HTTP 400, mapped to a per-question refusal HTTP 422 by our shim) at positions 65, 169, 474, 482, 515, 573, 580, 729, 833, 899, 951, 1057, 1116, 1198, 1418, identical to the v1 set; they count as wrong. The card also says Seb-9B-v2 was trained and evaluated with prompts of up to 1,024 tokens and that --max-model-len only sets the server limit; answered items run up to 1,875 tokens (mean 504, median 455), so some answered items are longer than the author's stated training and evaluation length. We used the card's 32,768 setting, as for v1. OWN HTTP SHIM: the wire layer is ours (see code_revision). TRAINING DATA AND SELF-REPORTED NUMBERS (the author's own card, unverified): the card reports Decision Index 0.3 public 59.16 (against 57.96 for hosted Jev 1.13 and 60.70 for Jev 1.14); this is not a JevBench result. We did not audit training data or overlap with the JevBench public 231 and ran no overlap check; no JevBench-use statement for v2 was recorded in the files read. HARDWARE: unlike v1 (A100 80 GB), this row ran on one RTX PRO 6000 (sm_120); latency reflects that card and the Gated-DeltaNet kernels ran clean (usage matched our offline count on all 1,485 answered items). 10 s RULE: max measured decision latency 0.138 s, 0 decisions over 10 s. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, max probability difference 0. No outside request for this model. PREDECESSOR: seb-9b (/jev-models/seb-9b); both measurements remain separate rows.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 57.38641643007268,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 5.4925826235513675
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 57.38641643007268
      },
      "calibration_gen": {
       "accuracy": 0.3564356435643564,
       "cc": 0.6457363133782437,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": -19.999999999999996,
        "score": 1.4706129698271897
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0.6457363133782437
      },
      "composite_gen": {
       "accuracy": 0.7623762376237624,
       "cc": 56.83985774960238,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": 33.33333333333333,
        "score": 63.20250007807544
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 56.83985774960238
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 78.79499429934914,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 76.38498289804741
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 78.79499429934914
      },
      "jaggedness": {
       "accuracy": 0.4388888888888889,
       "cc": 11.285698275626302,
       "cc_by_type": {
        "choice": 30.702598652550538,
        "noul": -35.13513513513513,
        "score": 38.2896313094635
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 11.285698275626302
      },
      "llm_router": {
       "accuracy": 0.8274760383386581,
       "cc": 71.57773436993652,
       "cc_by_type": {
        "choice": 81.77335563969358,
        "noul": 69.0721649484536,
        "score": 63.88768252166237
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 71.57773436993652
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -29.89981458777572,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100,
        "score": -15.472639639615803
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.68,
       "cc": 45.823066202819945,
       "cc_by_type": {
        "choice": 65.76459080534725,
        "noul": 25,
        "score": 46.70460780311259
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 45.823066202819945
      },
      "negation_pair_gen": {
       "accuracy": 0.627906976744186,
       "cc": 30.330265934353008,
       "cc_by_type": {
        "choice": 76.66666666666666,
        "noul": 0,
        "score": 14.32413113639237
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 30.330265934353008
      },
      "option_order": {
       "accuracy": 0.7916666666666666,
       "cc": 73.36884154460719,
       "cc_by_type": {
        "choice": 73.36884154460719
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 73.36884154460719
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 83.12848382106596,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100,
        "score": 73.4094754872219
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 83.12848382106596
      },
      "usecase": {
       "accuracy": 0.6227272727272727,
       "cc": 40.88753807810426,
       "cc_by_type": {
        "choice": 60.78431372549021,
        "noul": 16.666666666666675,
        "score": 45.211633842155884
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 40.88753807810426
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 4.49553438461395,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": -79.370539703301
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 4.49553438461395
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 55.80691686339742,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 54.824744445799034
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 55.80691686339742
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 68.11596222240989,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 53.2416919769642
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 68.11596222240989
      },
      "el": {
       "accuracy": 0.4,
       "cc": 22.079887607786528,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.94543563527765
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 22.079887607786528
      },
      "en": {
       "accuracy": 0.6533449174630755,
       "cc": 43.67778416055842,
       "cc_by_type": {
        "choice": 61.77737640497429,
        "noul": 25.179856115107913,
        "score": 44.076119961593044
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 43.67778416055842
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 64.52391114382674,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 100,
        "score": 22.935518097059315
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 64.52391114382674
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 48.30465437401824,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 44.36751503462304
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 48.30465437401824
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 82.37184101564326,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 89.46846422340035
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 82.37184101564326
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 18.888583868421673,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -50,
        "score": 40.18530467789072
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 18.888583868421673
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 85.61439800351773,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 70.87281209317207
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 85.61439800351773
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 32.49832518137121,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 19.999999999999996,
        "score": 34.683220182715694
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 32.49832518137121
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 27.528579980368807,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 27.03018438555087
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 27.528579980368807
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 93.79443805963497,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 98.04998084557158
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 93.79443805963497
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 18.84465359898142,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 56.53396079694426
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 18.84465359898142
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 30.861661394956045,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 27.81985511165611
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 30.861661394956045
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 73.76369289597659,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 63.71889247696926
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 73.76369289597659
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 33.91312705331635,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 19.999999999999996,
        "score": 58.33512584080012
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 33.91312705331635
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 36.4780826755024,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": -68.8266215387102
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 36.4780826755024
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 71.97310415505284,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 80.43544149741659
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 71.97310415505284
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 21.49023182592421,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": -68.8626378555607
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 21.49023182592421
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.712,
       "cc": 62.80066591400524,
       "cc_by_type": {
        "choice": 62.80066591400524
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 62.80066591400524
      },
      "noul": {
       "accuracy": 0.6293333333333333,
       "cc": 25.86666666666666,
       "cc_by_type": {
        "noul": 25.86666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 25.86666666666666
      },
      "score": {
       "accuracy": 0.584,
       "cc": 43.440386235078385,
       "cc_by_type": {
        "score": 43.440386235078385
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 43.440386235078385
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 22.47089665932023,
       "cc_by_type": {
        "choice": 52,
        "noul": 25,
        "score": -9.587310022039318
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 22.47089665932023
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": 1.9727743196350314,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -33.333333333333336,
        "score": -4.847722589749148
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 1.9727743196350314
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 68.48603888832874,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 87.97844186823825
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 68.48603888832874
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -7.921297584720989,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -33.333333333333336,
        "score": -8.559214391589865
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 33.47949224798697,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 62.757317323671046
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 33.47949224798697
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -44.26660553570567,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": 0,
        "score": -157.32811849390947
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 43.05378535028399,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 44.28740647102004
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 43.05378535028399
      },
      "gaming": {
       "accuracy": 0.4090909090909091,
       "cc": 4.9222426772100825,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50,
        "score": 37.93745973894732
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 4.9222426772100825
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 20.122783190160334,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 82.868349570481
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 20.122783190160334
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 33.38482637089782,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 0,
        "score": 38.69837632896756
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 33.38482637089782
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 48.04315272638223,
       "cc_by_type": {
        "choice": 100,
        "noul": 11.111111111111116,
        "score": 33.01834706803558
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 48.04315272638223
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -21.665428432461855,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": -69.29736056620277
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 46.309405333434256,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 19.999999999999996,
        "score": 53.47716033811467
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 46.309405333434256
      },
      "model_routing": {
       "accuracy": 0.847926267281106,
       "cc": 73.4346855974761,
       "cc_by_type": {
        "choice": 89.48051948051949,
        "noul": 78.78787878787878,
        "score": 52.035658524030026
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 73.4346855974761
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": 1.6124700570809363,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 50.90266928832534
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 1.6124700570809363
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 80.20609025286802,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100,
        "score": 57.331362680609644
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 80.20609025286802
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 27.437855304825547,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": 11.870216160782043
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 27.437855304825547
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 22.51872904218071,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 44.36778132944068
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 22.51872904218071
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.0851520625459,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.25545618763773
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.0851520625459
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.71560663717828,
      85.21517762668637
     ],
     "I": [
      39.989963245688045,
      50.10984922062006
     ],
     "capability": [
      59.63804732541866,
      66.85089580077984
     ],
     "composite": null,
     "seed": 70
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.5320529378626,
     "capability": 64.82241933825793,
     "intelligence": 46.11278573865326
    },
    "per_type": {
     "choice": {
      "accuracy": 0.712,
      "calibration": 80.5216524527087,
      "cc": 62.80066591400524,
      "cc_by_type": {
       "choice": 62.80066591400524
      },
      "cc_tiered": 65.5204398669211,
      "cc_tiered_open": 71.25353822050559,
      "cc_tiered_sealed": 59.787341513336614,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 62.80066591400524
     },
     "noul": {
      "accuracy": 0.6293333333333333,
      "calibration": 82.67294533051856,
      "cc": 25.86666666666666,
      "cc_by_type": {
       "noul": 25.86666666666666
      },
      "cc_tiered": 27.589604133574692,
      "cc_tiered_open": 38.33471359558317,
      "cc_tiered_sealed": 16.844494671566217,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 25.86666666666666
     },
     "score": {
      "accuracy": 0.584,
      "calibration": 87.40156103036057,
      "cc": 43.440386235078385,
      "cc_by_type": {
       "score": 43.440386235078385
      },
      "cc_tiered": 45.228313215463984,
      "cc_tiered_open": 50.55594929768438,
      "cc_tiered_sealed": 39.90067713324359,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.440386235078385
     }
    },
    "raw_on_A": {
     "calibration": 81.84002903415777,
     "capability": 65.22349885956152,
     "intelligence": 48.60696868496526
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 54.12546774850652,
    "Emphasis on Accuracy 60:20:20": 46.983990385240816,
    "Emphasis on Cost 20:20:60": 55.72754867943863,
    "Emphasis on Speed 20:60:20": 61.73418414680829,
    "Intelligence only": 39.22148829985125
   }
  },
  {
   "adapter_id": "images/semif-qwen3.5-4b/spec.json:remote_inproc_v15.RemoteInprocV15Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Theodore Lee (TheoLeeCJ)",
   "axes": {
    "calibration": 83.59960184071504,
    "cost": 77.61552273233744,
    "intelligence": 27.10113669038628,
    "speed": 90.43707331448911
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10233013188094879,
      "ece_score": 79.53397362381024,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 79.53397362381024,
      "score_before_time_limit": 79.53397362381024,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.38274932614555257,
      "brier": 0.1432523071034433,
      "ece": 0.08378278501118164,
      "n": 371,
      "score": 83.24344299776368,
      "score_before_time_limit": 83.24344299776368,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9701897018970189,
      "interval90_width": 0.6594399277326105,
      "n": 369,
      "nrps": 0.1212670700676031,
      "score": 88.0213889005712,
      "score_before_time_limit": 88.0213889005712,
      "time_limit_failed_share": 0,
      "top_ece": 0.059152575960486446
     }
    },
    "score": 83.59960184071504
   },
   "capability": 55.35036926555066,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 921,181 input + 0 output tokens over 1,624 decisions = 567.229679803 tokens/decision. GPU-time cost USD 0.00557377254813054/1,000; old list-price cost USD 0.0170168903940887/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00557377254813054/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005573772548130538
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SemIf, formerly OpenJev (Qwen3.5-4B, TheoLeeCJ)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 30.391025070826444,
    "I_sealed": 23.811248309946116,
    "base": 27.10113669038628,
    "excess": 4.006389118442085,
    "gap": 6.579776760880328,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 49.58254501782712,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 35.8288770053476,
       "judge": 60.44825313117996,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 14.439613526570046,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 13.043478260869556,
       "judge": -9.999999999999998,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 27.150916668082164,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.363221487255814,
       "hard": 12.79441887527647,
       "judge": 38.365166132705816,
       "standard": 21.436385647171228
      }
     },
     "sealed|choice": {
      "cc": 45.71562508055857,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 61.4219200354741,
       "hard": 39.93212669683257,
       "judge": 45.99627560521416,
       "standard": 49.008498583569406
      }
     },
     "sealed|noul": {
      "cc": 2.17292780700555,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 43.99999999999999,
       "hard": -22.330097087378643,
       "judge": 2.6315789473684292,
       "standard": 29.5774647887324
      }
     },
     "sealed|score": {
      "cc": 23.545192042274238,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.77035344302047,
       "hard": 13.542770599427534,
       "judge": 22.665102695300575,
       "standard": 24.757588248055008
      }
     }
    }
   },
   "jevbench_score": 16.141782263417074,
   "key": "semif-qwen3.5-4b",
   "last_measured_on": "2026-10-02",
   "licence": "MIT (code); Qwen3.5 weights Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9039301310043668,
    "decisive_rate": 0.6172506738544474,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.59960184071504,
     "cost": 63.07359395281681,
     "intelligence": 27.10113669038628,
     "speed": 90.76560119421038
    },
    "board_release": "v1.6.43",
    "capability": 55.35036926555066,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 81,
    "capability_rank": 107,
    "composite_ci95": {
     "A": [
      9.244572028428712,
      23.18413753262927
     ],
     "B": [
      7.522852111652043,
      19.995799931382237
     ],
     "C": [
      6.419841686408828,
      16.100095508770327
     ]
    },
    "jevbench_score": 15.517179498801362,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 121,
    "rank": 159,
    "ranks": {
     "A": 159,
     "B": 160,
     "C": 160,
     "capability": 162
    },
    "scores": {
     "A": 15.517179498801362,
     "B": 13.04205072010579,
     "C": 10.775819096389835
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.25408456099685284,
     "p50_s_raw": 0.05204228049842641,
     "p95_s_adjusted": 0.32996023471932856,
     "p95_s_raw": 0.08998011735966428
    },
    "views": {
     "Balanced 33:33:33": 13.820858864848555,
     "Emphasis on Accuracy 60:20:20": 10.67791560389241,
     "Emphasis on Cost 20:20:60": 15.384875177789313,
     "Emphasis on Speed 20:60:20": 17.119455792629843,
     "Intelligence only": 7.962006198162448
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 200,
   "ranked": true,
   "ranks": {
    "A": 200,
    "B": 205,
    "C": 200,
    "capability": 196
   },
   "repo": "https://github.com/TheoLeeCJ/openjev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.141782263417074,
    "B": 13.390444587593713,
    "C": 11.209571016261856
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26757615598617124,
    "p50_s_raw": 0.05878807799308561,
    "p95_s_adjusted": 0.337944403081201,
    "p95_s_raw": 0.0939722015406005
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B (frozen, BF16)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.25331982816681,
      85.80759674871754
     ],
     "I": [
      22.076409045997522,
      31.830294818443246
     ],
     "capability": [
      51.05521406162153,
      58.22417544189523
     ],
     "composite": null,
     "seed": 99
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.59960184071504,
     "capability": 55.35036926555066,
     "intelligence": 27.10113669038628
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5893333333333334,
      "calibration": 79.53397362381024,
      "cc": 47.48159866619932,
      "cc_by_type": {
       "choice": 47.48159866619932
      },
      "cc_tiered": 47.64908504919285,
      "cc_tiered_open": 49.58254501782712,
      "cc_tiered_sealed": 45.71562508055857,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 47.48159866619932
     },
     "noul": {
      "accuracy": 0.552,
      "calibration": 83.24344299776368,
      "cc": 10.40000000000001,
      "cc_by_type": {
       "noul": 10.40000000000001
      },
      "cc_tiered": 8.306270666787798,
      "cc_tiered_open": 14.439613526570046,
      "cc_tiered_sealed": 2.17292780700555,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 10.40000000000001
     },
     "score": {
      "accuracy": 0.49333333333333335,
      "calibration": 88.0213889005712,
      "cc": 25.84341933063762,
      "cc_by_type": {
       "score": 25.84341933063762
      },
      "cc_tiered": 25.3480543551782,
      "cc_tiered_open": 27.150916668082164,
      "cc_tiered_sealed": 23.545192042274238,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 25.84341933063762
     }
    },
    "raw_on_A": {
     "calibration": 83.04331750428547,
     "capability": 58.219039699698534,
     "intelligence": 33.3947618951116
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 14.486558994357244,
    "Emphasis on Accuracy 60:20:20": 10.910325060036358,
    "Emphasis on Cost 20:20:60": 16.960768135266033,
    "Emphasis on Speed 20:60:20": 17.707694857446334,
    "Intelligence only": 7.962006198162448
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "shisa-ai",
   "axes": {
    "calibration": 81.16385348785812,
    "cost": 71.1478434006942,
    "intelligence": 49.90370340622748,
    "speed": 92.69330974783162
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09572306078200861,
      "ece_score": 80.85538784359828,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.74758065980681,
      "score_before_time_limit": 80.85538784359828,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.24128686327077753,
      "brier": 0.1296093675410914,
      "ece": 0.09211681556391924,
      "n": 373,
      "score": 81.57663688721615,
      "score_before_time_limit": 81.57663688721615,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9005376344086021,
      "interval90_width": 0.3373655913978495,
      "n": 372,
      "nrps": 0.10181393375418091,
      "score": 81.1673429165514,
      "score_before_time_limit": 81.1673429165514,
      "time_limit_failed_share": 0,
      "top_ece": 0.1374196039573955
     }
    },
    "score": 81.16385348785812
   },
   "capability": 65.5337784470428,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 779,620 input + 0 output tokens over 1,485 decisions = 524.996632997 tokens/decision. GPU-time cost USD 0.00915669088398183/1,000; old list-price cost USD 0.047249696969697/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00915669088398183/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.046777200000000005
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009156690883981826
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "shisa-de-1 (shisa-ai, Gemma-4-26B-A4B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Stock vLLM server read through the evaluator's shisa_vllm.py adapter.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 61.18017895681861,
    "I_sealed": 41.00946236886275,
    "base": 51.094820662840675,
    "excess": 10.331189817600148,
    "gap": 20.17071658795586,
    "penalty": 0.9766881018239986,
    "per_type_split": {
     "open|choice": {
      "cc": 74.65954284154392,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 67.9144385026738,
       "judge": 68.35860250494397,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 59.19806763285025,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 49.68292639606169,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.25970491817685,
       "hard": 48.94091823006358,
       "judge": 53.33457095279339,
       "standard": 36.90108663190276
      }
     },
     "sealed|choice": {
      "cc": 63.542627740178986,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 57.32624031455995,
       "judge": 63.85896180215475,
       "standard": 61.08415946002954
      }
     },
     "sealed|noul": {
      "cc": 25.330413282034485,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 55.172413793103445,
       "hard": 9.090909090909083,
       "judge": 23.076923076923084,
       "standard": 46.26865671641791
      }
     },
     "sealed|score": {
      "cc": 34.155346084374784,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 59.91120064082291,
       "hard": 23.49152484245691,
       "judge": 33.260433360671,
       "standard": 43.94743037554214
      }
     }
    }
   },
   "jevbench_score": 69.6574933188226,
   "key": "shisa-de-1",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "shisa-ai/shisa-de-1@9111f4746a96d6cdf9f1a23a88afd5b57244508b",
   "noul_decisive": {
    "acc_among_decisive": 0.901060070671378,
    "decisive": 283,
    "decisive_rate": 0.7587131367292225,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.19978921578861,
     "cost": 49.76802917325137,
     "intelligence": 49.89769839508435,
     "speed": 92.81832911721798
    },
    "board_release": "v1.6.43",
    "capability": 65.54874380543649,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 36,
    "capability_rank": 57,
    "composite_ci95": {
     "A": [
      46.9264437048365,
      64.2980956074595
     ],
     "B": [
      43.73146315925963,
      61.7198788686833
     ],
     "C": [
      32.587808128358674,
      51.81649202784387
     ]
    },
    "jevbench_score": 62.425924800405696,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": 32,
    "rank": 54,
    "ranks": {
     "A": 54,
     "B": 53,
     "C": 60,
     "capability": 99
    },
    "scores": {
     "A": 62.425924800405696,
     "B": 59.25066612002561,
     "C": 43.35133666694839
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2114860150963068,
     "p50_s_raw": 0.0307430075481534,
     "p95_s_adjusted": 0.24710721960291265,
     "p95_s_raw": 0.04855360980145633
    },
    "views": {
     "Balanced 33:33:33": 58.14568090369809,
     "Emphasis on Accuracy 60:20:20": 54.21985915007682,
     "Emphasis on Cost 20:20:60": 54.15770034691774,
     "Emphasis on Speed 20:60:20": 68.08963018332004,
     "Intelligence only": 49.23369255351228
    }
   },
   "rank": 82,
   "ranked": true,
   "ranks": {
    "A": 82,
    "B": 98,
    "C": 105,
    "capability": 128
   },
   "repo": "https://huggingface.co/shisa-ai/shisa-de-1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.6574933188226,
    "B": 64.48298754927248,
    "C": 48.37325924918236
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21332498043775558,
    "p50_s_raw": 0.03166249021887779,
    "p95_s_adjusted": 0.25213161795400085,
    "p95_s_raw": 0.051065808977000415
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 65.92917797912096,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 6.120867270696206
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 65.92917797912096
      },
      "calibration_gen": {
       "accuracy": 0.6237623762376238,
       "cc": 40.3429206482318,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": 19.999999999999996,
        "score": 51.930140523698576
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 40.3429206482318
      },
      "composite_gen": {
       "accuracy": 0.5445544554455446,
       "cc": 21.424225638719204,
       "cc_by_type": {
        "choice": 70.73170731707317,
        "noul": -18.518518518518523,
        "score": 12.059488117602957
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 21.424225638719204
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.41167071307218,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 90.23501213921652
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.41167071307218
      },
      "jaggedness": {
       "accuracy": 0.5444444444444444,
       "cc": 20.239942948437324,
       "cc_by_type": {
        "choice": 55.24542829643889,
        "noul": -8.108108108108103,
        "score": 13.582508656981185
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 20.239942948437324
      },
      "llm_router": {
       "accuracy": 0.805111821086262,
       "cc": 70.45274745673682,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 79.38144329896907,
        "score": 62.88284820393485
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 70.45274745673682
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5925925925925926,
       "cc": 16.255716229910213,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": -25,
        "score": 10.880550751586304
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 16.255716229910213
      },
      "multilingual": {
       "accuracy": 0.7090909090909091,
       "cc": 52.883240812377124,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": 38.888888888888886,
        "score": 50.08362454922064
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 52.883240812377124
      },
      "negation_pair_gen": {
       "accuracy": 0.6511627906976745,
       "cc": 18.559880414922972,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 13.33333333333333,
        "score": -40.98702542189774
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 18.559880414922972
      },
      "option_order": {
       "accuracy": 0.8125,
       "cc": 76.03195739014647,
       "cc_by_type": {
        "choice": 76.03195739014647
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 76.03195739014647
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 78.19555571499772,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 100,
        "score": 70.6227031810292
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 78.19555571499772
      },
      "usecase": {
       "accuracy": 0.6181818181818182,
       "cc": 39.882819009244145,
       "cc_by_type": {
        "choice": 64.4607843137255,
        "noul": 20.833333333333325,
        "score": 34.35433938067361
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 39.882819009244145
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 24.773578108952535,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50,
        "score": -56.63164662552334
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 24.773578108952535
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 51.17586100498403,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 100,
        "score": -4.997762607167711
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 51.17586100498403
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 88.18127171952362,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 88.43762046830534
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 88.18127171952362
      },
      "el": {
       "accuracy": 0.6,
       "cc": 49.62590138419713,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 0,
        "score": 99.15703376152992
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 49.62590138419713
      },
      "en": {
       "accuracy": 0.6707211120764552,
       "cc": 44.90753439146581,
       "cc_by_type": {
        "choice": 64.85275991262004,
        "noul": 33.09352517985611,
        "score": 36.776318081921254
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 44.90753439146581
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 61.28889199968085,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": -6.345395779097762
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 61.28889199968085
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 22.258623278201583,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 98.46985890564301
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 22.258623278201583
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 75.251652634066,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 68.10789907866861
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 75.251652634066
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 52.322008035025284,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 25,
        "score": 48.725800641388695
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 52.322008035025284
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 51.875208114849244,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": -16.315139490214527
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 51.875208114849244
      },
      "ja": {
       "accuracy": 0.6,
       "cc": 29.72773865128922,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 0,
        "score": 32.07439943281921
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 29.72773865128922
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 36.782423082121106,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 0,
        "score": 54.791713690807754
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 36.782423082121106
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.77110855481779,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.31332566445336
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.77110855481779
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 39.018116704884726,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 35.669068729372796
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 39.018116704884726
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.717948717948718,
       "cc": 45.24014195316485,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 0,
        "score": 43.33786132288856
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 45.24014195316485
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 76.2974441721712,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100,
        "score": 50.106239411033336
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 76.2974441721712
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 34.42500577685311,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 19.999999999999996,
        "score": 34.33884711779338
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 34.42500577685311
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 61.5543127478497,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50,
        "score": 56.40206867833171
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 61.5543127478497
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 75.41645146226901,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 90.76548341906509
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 75.41645146226901
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 28.838327496661037,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": -46.81835084335022
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 28.838327496661037
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.74,
       "cc": 66.41726783903252,
       "cc_by_type": {
        "choice": 66.41726783903252
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 66.41726783903252
      },
      "noul": {
       "accuracy": 0.68,
       "cc": 36.00000000000001,
       "cc_by_type": {
        "noul": 36.00000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 36.00000000000001
      },
      "score": {
       "accuracy": 0.5733333333333334,
       "cc": 38.69319904008759,
       "cc_by_type": {
        "score": 38.69319904008759
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 38.69319904008759
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 0.8000025898825479,
       "cc_by_type": {
        "choice": 68,
        "noul": -25,
        "score": -40.599992230352356
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0.8000025898825479
      },
      "code_linting": {
       "accuracy": 0.6666666666666666,
       "cc": 42.720119762463035,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 33.33333333333333,
        "score": 13.460566326726564
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 42.720119762463035
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 27.17283322265492,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": -35.96117512878321
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 27.17283322265492
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -14.22728275682919,
       "cc_by_type": {
        "choice": -5.263157894736844,
        "noul": -33.333333333333336,
        "score": -4.085357042417392
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 31.17848786632922,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 41.36155055550942
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 31.17848786632922
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -11.245859971103116,
       "cc_by_type": {
        "choice": 62.264150943396224,
        "noul": -33.333333333333336,
        "score": -62.668397523372235
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 35.45895960403414,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": 14.28571428571428,
        "score": 41.110772369525414
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 35.45895960403414
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": -2.1173603271195396,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -100,
        "score": 45.91272738101071
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 24.81760578587952,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 16.95281735763855
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 24.81760578587952
      },
      "knowledge_graphs": {
       "accuracy": 0.5,
       "cc": 31.67389628464957,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 13.080218475647499
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 31.67389628464957
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 58.062709443665405,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 8.038257529962634
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 58.062709443665405
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 59.72549296402523,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100,
        "score": 75.9506724404628
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 59.72549296402523
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 36.95747791084126,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -19.999999999999996,
        "score": 42.38874851179442
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 36.95747791084126
      },
      "model_routing": {
       "accuracy": 0.8709677419354839,
       "cc": 77.7271211623311,
       "cc_by_type": {
        "choice": 83.63636363636363,
        "noul": 96.96969696969697,
        "score": 52.575302880932725
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 77.7271211623311
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 38.84787678865832,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 0,
        "score": 51.09257470378685
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 38.84787678865832
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 52.02607140073403,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 39.45797279087437
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 52.02607140073403
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 20.93395620361999,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": 0,
        "score": 21.9151691034708
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 20.93395620361999
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 25.670072643901648,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 0,
        "score": 34.98123242445856
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 25.670072643901648
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 92.24259919119889,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 87.40395415722656
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 92.24259919119889
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.8995944023424,
      83.8973143336525
     ],
     "I": [
      44.09204975421906,
      53.92796116673747
     ],
     "capability": [
      61.00415898154735,
      68.10241136551379
     ],
     "composite": null,
     "seed": 70
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.16385348785812,
     "capability": 65.5337784470428,
     "intelligence": 49.90370340622748
    },
    "per_type": {
     "choice": {
      "accuracy": 0.74,
      "calibration": 80.74758065980681,
      "cc": 66.41726783903252,
      "cc_by_type": {
       "choice": 66.41726783903252
      },
      "cc_tiered": 69.10108529086145,
      "cc_tiered_open": 74.65954284154392,
      "cc_tiered_sealed": 63.542627740178986,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 66.41726783903252
     },
     "noul": {
      "accuracy": 0.68,
      "calibration": 81.57663688721615,
      "cc": 36.00000000000001,
      "cc_by_type": {
       "noul": 36.00000000000001
      },
      "cc_tiered": 42.26424045744237,
      "cc_tiered_open": 59.19806763285025,
      "cc_tiered_sealed": 25.330413282034485,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 36.00000000000001
     },
     "score": {
      "accuracy": 0.5733333333333334,
      "calibration": 81.1673429165514,
      "cc": 38.69319904008759,
      "cc_by_type": {
       "score": 38.69319904008759
      },
      "cc_tiered": 41.919136240218236,
      "cc_tiered_open": 49.68292639606169,
      "cc_tiered_sealed": 34.155346084374784,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 38.69319904008759
     }
    },
    "raw_on_A": {
     "calibration": 78.9590048315948,
     "capability": 64.27027081646067,
     "intelligence": 49.581536801326536
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.58457776590478,
    "Emphasis on Accuracy 60:20:20": 58.62524969768195,
    "Emphasis on Cost 20:20:60": 68.23651664255388,
    "Emphasis on Speed 20:60:20": 74.94526097059436,
    "Intelligence only": 49.711666243536136
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "sthanika-ai (Hugging Face)",
   "axes": {
    "calibration": 93.43694381196171,
    "cost": 56.225182205710844,
    "intelligence": 68.69375926172683,
    "speed": 85.61790755169669
   },
   "base_model": "google/gemma-4-31B-it",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.02447737704917985,
      "ece_score": 95.10452459016403,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 95.10452459016403,
      "score_before_time_limit": 95.10452459016403,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.1769436997319035,
      "brier": 0.05653745651474539,
      "ece": 0.043094906166219714,
      "n": 373,
      "score": 91.38101876675606,
      "score_before_time_limit": 91.38101876675606,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.978494623655914,
      "interval90_width": 0.2703853046594983,
      "n": 372,
      "nrps": 0.051695313689516104,
      "score": 93.82528807896506,
      "score_before_time_limit": 93.82528807896506,
      "time_limit_failed_share": 0,
      "top_ece": 0.035899462365591404
     }
    },
    "score": 93.43694381196171
   },
   "capability": 81.06535153684428,
   "code_revision": "Author's Sieve library at 3ff9195c902ee1882e3e4bcfcb5bc733b8fcbb7f (sieve-decisions 0.3.3) and decider-ai 1.6.0, served through our own shim: a forward-only wrapper around the author's SlotsEngine. The shim passes state and questions unchanged and returns native answers and usage unchanged, replacing only the model label. Native unsupported requests become HTTP 422; other exceptions stop the run. No evaluator prompt building, normalisation, truncation, selection, calibration or post-processing. Author defaults: chat layout, thinking off, one independent answer slot per question, option-label logits with softmax temperature 8.5495, bidirectional global attention, max_tokens 32,768, fwd_tokens 16,384, full_release True. BF16 base plus adapter containing the full fine-tuned decoder layers; no merged checkpoint or quantisation. Base google/gemma-4-31B-it at 842da3794eaa0b77d5f08bae87a17459d91ff475; adapter at 2c0d64310ac944b350a9e19f1f54dcd9a5bed3eb. Source reviewed before measurement; author code ran in an offline disposable environment. Runtime: torch 2.8.0, transformers 5.17.0, CUDA 12.8.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 714,573 input + 0 output tokens over 1,485 decisions = 481.193939394 tokens/decision. GPU-time cost USD 0.0287846261407634/1,000; old list-price cost USD 0.0433074545454545/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0287846261407634/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.042874380000000004
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.028784626140763356
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "First measurement of Sieve-31B-v2; one new self-hosted row replacing nothing. Sieve-31B v1 was not measured here. Full fine-tune of 60 text decoder layers of google/gemma-4-31B-it, stored as a PEFT adapter. No shared adapter LFS objects across 15 compared repositories; this establishes byte difference only, not independent training. Jev-compatible only. Our own shim is a forward-only wrapper around the author's engine, with no evaluator prompt building, selection or post-processing. Native choice, noul and score probabilities use the author's declared training and serving temperature 8.5495. The training configuration identifies that temperature in the training objective and at inference; it does not identify a separate post-hoc fit on JevBench public items. Training datasets are not identified. We did not inspect training examples or check overlap with public or sealed JevBench items; contamination is unchecked. The author's public evaluation is Decision Index 0.3, a separate benchmark. The native formatter normalises each distribution and rounds probabilities to 4 decimals; maximum observed returned sum error 0.0002. Author defaults were used; no other setting measured. The engine's 32,768-token limit natively refused 15 preregistered long items before GPU work, without truncation. All 15 returned HTTP 422 and were scored as misses; the other 1,485 of 1,500 returned HTTP 200. No unexpected failure. Zero decisions exceeded 10 seconds; the 15 context refusals remain misses. Loopback p50 0.138 s, p95 0.248 s, maximum 0.475 s; adjusted common self-hosted p50 0.425271 s. Native input usage 714,573 equals the offline count; output usage is zero. Determinism: 50/50 requests bit-identical, no argmax flips. One attempt, no discarded scored attempt. Cost is an ESTIMATE from the base-model market reference.",
   "display": "Sieve-31B-v2",
   "draw": {
    "id": "v1.6-regular-20261012",
    "measured_on": "2026-10-25",
    "release": "v1.7.5",
    "results_sha256": "db2289a36c22eac5ac03125690c2c9dbb9d1b024e662d17beb80e240bdd85859"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB, Finland, EU. One attempt, measured 25 Oct 2026. Author engine behind our own loopback shim in an offline, read-only container with empty environment and hash-checked model and code. Complete 1,500-request run; scored loop 237.8 s. About USD 1.31 GPU rental, separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.37372405803248,
    "I_sealed": 65.01379446542117,
    "base": 68.69375926172683,
    "excess": -2.4795971777444024,
    "gap": 7.359929592611309,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 87.38389224457799,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 88.13447593935399,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 66.46997929606626,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 50,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 63.267300633453175,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 74.42878787878789,
       "hard": 75.80490740740741,
       "judge": 55.45881355932203,
       "standard": 44.32407407407406
      }
     },
     "sealed|choice": {
      "cc": 79.7233941732884,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 75.5406499363941,
       "judge": 81.48873653281098,
       "standard": 80.06749630879561
      }
     },
     "sealed|noul": {
      "cc": 54.702583057704004,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 75.86206896551724,
       "hard": 45.45454545454546,
       "judge": 47.6923076923077,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 60.615406165271125,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 77.93307017543859,
       "hard": 57.058,
       "judge": 59.81390624999998,
       "standard": 60.27363636363636
      }
     }
    }
   },
   "jevbench_score": 73.0924793557567,
   "key": "sieve-31b-v2",
   "last_measured_on": "2026-10-25",
   "licence": "Apache-2.0 per model card and LICENSE on weights and author library; base google/gemma-4-31B-it public and ungated under its own terms. No serving code redistributed here.",
   "listing": "ranked",
   "measured_in": "v1.7.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "sthanika-ai/Sieve-31B-v2@2c0d64310ac944b350a9e19f1f54dcd9a5bed3eb + google/gemma-4-31B-it@842da3794eaa0b77d5f08bae87a17459d91ff475",
   "new_in": "v1.7.5",
   "noul_decisive": {
    "acc_among_decisive": 0.9771986970684039,
    "decisive": 307,
    "decisive_rate": 0.8230563002680965,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 47,
   "ranked": true,
   "ranks": {
    "A": 47,
    "B": 35,
    "C": 38,
    "capability": 29
   },
   "repo": "https://huggingface.co/sthanika-ai/Sieve-31B-v2",
   "scores": {
    "A": 73.0924793557567,
    "B": 72.16823853847822,
    "C": 73.0924793557567
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.42527127700000167,
    "p50_s_raw": 0.13763563850000082,
    "p95_s_adjusted": 0.6449754638998229,
    "p95_s_raw": 0.24748773194991144
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 84.50222222222224,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 61.84000000000003
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 84.50222222222224
      },
      "calibration_gen": {
       "accuracy": 0.7821782178217822,
       "cc": 65.25852023683281,
       "cc_by_type": {
        "choice": 74.54931071049842,
        "noul": 60.00000000000001,
        "score": 61.22624999999999
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 65.25852023683281
      },
      "composite_gen": {
       "accuracy": 0.9405940594059405,
       "cc": 89.2437399532115,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 81.64269230769231
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 89.2437399532115
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.69333333333333,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.08
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.69333333333333
      },
      "jaggedness": {
       "accuracy": 0.65,
       "cc": 47.117142813424316,
       "cc_by_type": {
        "choice": 59.57651588065449,
        "noul": 24.32432432432432,
        "score": 57.45058823529412
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 47.117142813424316
      },
      "llm_router": {
       "accuracy": 0.9137380191693291,
       "cc": 81.40583530059047,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 93.81443298969072,
        "score": 55.950312500000024
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 81.40583530059047
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5925925925925926,
       "cc": 26.28461790214367,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -25,
        "score": 65.7095238095238
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 26.28461790214367
      },
      "multilingual": {
       "accuracy": 0.7454545454545455,
       "cc": 57.14253696313542,
       "cc_by_type": {
        "choice": 78.4805999347897,
        "noul": 30.555555555555557,
        "score": 62.391455399061016
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 57.14253696313542
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 74.80122222222224,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 57.737000000000016
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.80122222222224
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.921875,
       "cc": 88.6168687918688,
       "cc_by_type": {
        "choice": 90.99099099099098,
        "noul": 100,
        "score": 74.85961538461538
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 88.6168687918688
      },
      "usecase": {
       "accuracy": 0.7454545454545455,
       "cc": 58.260522091503276,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 37.5,
        "score": 60.56588000000001
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 58.260522091503276
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 43.949560439560436,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 50,
        "score": 38.99153846153845
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 43.949560439560436
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 58.32900204813109,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 62.391
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 58.32900204813109
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 90.70803129408746,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 100,
        "score": 80.08869565217391
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 90.70803129408746
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 61.0096679081316,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 33.33333333333333,
        "score": 99.97500000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 61.0096679081316
      },
      "en": {
       "accuracy": 0.8262380538662033,
       "cc": 71.41921016592983,
       "cc_by_type": {
        "choice": 82.42637995631002,
        "noul": 68.34532374100719,
        "score": 63.48592680047227
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 71.41921016592983
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 66.8387298083997,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 10.304117647058831
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.8387298083997
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.75999999999999,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.94666666666664
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.75999999999999
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 75.17107327141383,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 89.04263157894736
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 75.17107327141383
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 39.543065642458096,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 52.148749999999986
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 39.543065642458096
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 50.65155587722228,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -34.0157142857143
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 50.65155587722228
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 48.20009072456094,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 39.99999999999999,
        "score": 18.897333333333343
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 48.20009072456094
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 24.436666666666667,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 73.30999999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 24.436666666666667
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.63944444444445,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.58500000000001
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.63944444444445
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -2.358008658008657,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 18.899999999999995
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 66.96558352770587,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 39.99999999999999,
        "score": 68.51418604651163
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 66.96558352770587
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 50.80992827317024,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 19.999999999999996,
        "score": 64.25064516129034
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 50.80992827317024
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 57.98,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 53.94000000000001
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 57.98
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 38.2223731884058,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50,
        "score": -13.593750000000004
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 38.2223731884058
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 67.42129032258065,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 66.78
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.42129032258065
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 54.56777777777777,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 30.370000000000008
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 54.56777777777777
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.856,
       "cc": 81.40033295700262,
       "cc_by_type": {
        "choice": 81.40033295700262
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.40033295700262
      },
      "noul": {
       "accuracy": 0.8,
       "cc": 60.00000000000001,
       "cc_by_type": {
        "noul": 60.00000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 60.00000000000001
      },
      "score": {
       "accuracy": 0.72,
       "cc": 62.2710559566787,
       "cc_by_type": {
        "score": 62.2710559566787
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 62.2710559566787
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 14.073725490196077,
       "cc_by_type": {
        "choice": 68,
        "noul": -50,
        "score": 24.221176470588233
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 14.073725490196077
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 82.81515669515669,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 70.6676923076923
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 82.81515669515669
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 42.729945799457994,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50,
        "score": -5.5500000000000105
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 42.729945799457994
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -12.227458576998046,
       "cc_by_type": {
        "choice": 53.21637426900585,
        "noul": -100,
        "score": 10.101250000000018
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 59.776358695652185,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 74.98125
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.776358695652185
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": 24.249067085953897,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": 66.66666666666667,
        "score": -43.60499999999994
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 24.249067085953897
      },
      "financial_crime": {
       "accuracy": 0.5666666666666667,
       "cc": 27.236372189901598,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -42.85714285714286,
        "score": 44.174102564102554
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 27.236372189901598
      },
      "gaming": {
       "accuracy": 0.6818181818181818,
       "cc": 31.658632530424683,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -50,
        "score": 76.33478260869565
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 31.658632530424683
      },
      "insurance_claims": {
       "accuracy": 0.6,
       "cc": 39.114259259259256,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 19.999999999999996,
        "score": 39.00944444444444
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 39.114259259259256
      },
      "knowledge_graphs": {
       "accuracy": 0.8181818181818182,
       "cc": 74.62628414768672,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100,
        "score": 36.726818181818174
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 74.62628414768672
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 53.99059666431759,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 40.266363636363636
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 53.99059666431759
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 9.062213261648742,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -9.372500000000006
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 9.062213261648742
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 59.43456174024311,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 69.81999999999998
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.43456174024311
      },
      "model_routing": {
       "accuracy": 0.8894009216589862,
       "cc": 77.26560121302309,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 93.93939393939394,
        "score": 42.53273437499999
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 77.26560121302309
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 20.022848970682332,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 83.10117647058823
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 20.022848970682332
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.09353700926124,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 62.37346153846154
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.09353700926124
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 11.78775330821984,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -50,
        "score": 29.698235294117648
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 11.78775330821984
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 16.491381642512078,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": -33.333333333333336,
        "score": 69.764
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 16.491381642512078
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.68464912280702,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.05394736842105
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.68464912280702
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      89.04466015908608,
      93.49089610112732
     ],
     "I": [
      63.69414320468904,
      72.651763833468
     ],
     "capability": [
      77.10403197521909,
      82.45099585649031
     ],
     "composite": null,
     "seed": 71
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 93.43694381196171,
     "capability": 81.06535153684428,
     "intelligence": 68.69375926172683
    },
    "per_type": {
     "choice": {
      "accuracy": 0.856,
      "calibration": 95.10452459016403,
      "cc": 81.40033295700262,
      "cc_by_type": {
       "choice": 81.40033295700262
      },
      "cc_tiered": 83.5536432089332,
      "cc_tiered_open": 87.38389224457799,
      "cc_tiered_sealed": 79.7233941732884,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.40033295700262
     },
     "noul": {
      "accuracy": 0.8,
      "calibration": 91.38101876675606,
      "cc": 60.00000000000001,
      "cc_by_type": {
       "noul": 60.00000000000001
      },
      "cc_tiered": 60.58628117688513,
      "cc_tiered_open": 66.46997929606626,
      "cc_tiered_sealed": 54.702583057704004,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 60.00000000000001
     },
     "score": {
      "accuracy": 0.72,
      "calibration": 93.82528807896506,
      "cc": 62.2710559566787,
      "cc_by_type": {
       "score": 62.2710559566787
      },
      "cc_tiered": 61.94135339936215,
      "cc_tiered_open": 63.267300633453175,
      "cc_tiered_sealed": 60.615406165271125,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.2710559566787
     }
    },
    "raw_on_A": {
     "calibration": 91.56748908957091,
     "capability": 80.09510945628202,
     "intelligence": 68.62272982299311
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.14652507327233,
    "Emphasis on Accuracy 60:20:20": 68.36436914218804,
    "Emphasis on Cost 20:20:60": 62.818775926294485,
    "Emphasis on Speed 20:60:20": 74.20336738509468,
    "Intelligence only": 68.69375926172683
   }
  },
  {
   "adapter_id": "images/simplejev-qwen3.5-0.8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "sabeel111 / Featherless AI",
   "axes": {
    "calibration": 49.46821562395294,
    "cost": 76.868497665666,
    "intelligence": 12.67752479561282,
    "speed": 58.64501925086723
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.22436826460136267,
      "ece_score": 55.126347079727466,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 54.832339895302255,
      "score_before_time_limit": 55.126347079727466,
      "time_limit_failed_share": 0.005333333333333333
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.40897819205718194,
      "ece": 0.4071351151116132,
      "n": 371,
      "score": 18.523449039070215,
      "score_before_time_limit": 18.572976977677357,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.8807588075880759,
      "interval90_width": 0.6312104787714541,
      "n": 369,
      "nrps": 0.17353025104683517,
      "score": 75.04885793748636,
      "score_before_time_limit": 76.89432165726062,
      "time_limit_failed_share": 0.024,
      "top_ece": 0.14429165790397622
     }
    },
    "score": 49.46821562395294
   },
   "capability": 31.07287020978288,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.8B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,585,602 input + 0 output tokens over 1,624 decisions = 976.35591133 tokens/decision. GPU-time cost USD 0.00590269139250528/1,000; old list-price cost USD 0.00976355911330049/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00590269139250528/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005902691392505279
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 14,
    "n_over_limit_answered_ok": 14,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SimpleJev (Qwen3.5-0.8B, CPU)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 16.855956156782675,
    "I_sealed": 8.499093434442965,
    "base": 12.67752479561282,
    "excess": 5.783475079901466,
    "gap": 8.35686272233971,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 16.101545239125173,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 2.702702702702708,
       "hard": 13.180245360176167,
       "judge": 28.80685563612393,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 31.332643202208423,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 30.434782608695656,
       "judge": 30.000000000000004,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 3.1336800290144353,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 21.659111114444617,
       "hard": -15.769090377553162,
       "judge": 24.5874654817379,
       "standard": -0.504172879650655
      }
     },
     "sealed|choice": {
      "cc": 16.18809910474239,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 10.87133249575052,
       "hard": 9.389140271493197,
       "judge": 19.92551210428307,
       "standard": 26.838280576425678
      }
     },
     "sealed|noul": {
      "cc": 9.309181198586508,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 24,
       "hard": 12.621359223300965,
       "judge": 5.263157894736836,
       "standard": 1.4084507042253502
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 19.829285077433635,
       "hard": -17.18882511862938,
       "judge": 2.232240541966324,
       "standard": -0.6083842924782035
      }
     }
    }
   },
   "jevbench_score": 1.991018277559104,
   "key": "simplejev-qwen3.5-0.8b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 server; Apache-2.0 Qwen3.5-0.8B checkpoint",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.581081081081081,
    "decisive_rate": 1,
    "supported": true,
    "valid": 370
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 50.197881904888476,
     "cost": 70.31175519898045,
     "intelligence": 13.000426205687345,
     "speed": 58.929227255635794
    },
    "board_release": "v1.6.43",
    "capability": 31.59915405528791,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.709004207843539,
      4.446455346949053
     ],
     "B": [
      0.5274465681104142,
      3.6235557003922736
     ],
     "C": [
      0.49236403322467986,
      3.0878162131590647
     ]
    },
    "jevbench_score": 2.112084174105171,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 155,
    "rank": 197,
    "ranks": {
     "A": 197,
     "B": 197,
     "C": 197,
     "capability": 225
    },
    "scores": {
     "A": 2.112084174105171,
     "B": 1.649257467523907,
     "C": 1.4667251209063683
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 8.797023160476238,
     "p50_s_raw": 4.323511580238119,
     "p95_s_adjusted": 14.545931534096598,
     "p95_s_raw": 7.197965767048299
    },
    "views": {
     "Balanced 33:33:33": 1.8759479039856477,
     "Emphasis on Accuracy 60:20:20": 1.2903897195924547,
     "Emphasis on Cost 20:20:60": 2.475315897383538,
     "Emphasis on Speed 20:60:20": 2.3795755523953557,
     "Intelligence only": 0.8788864373471849
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 237,
   "ranked": true,
   "ranks": {
    "A": 237,
    "B": 237,
    "C": 237,
    "capability": 267
   },
   "repo": "https://github.com/featherless-ai/simple-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 1.991018277559104,
    "B": 1.5451175477503902,
    "C": 1.3826515816382667
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 9.16144129736349,
    "p50_s_raw": 4.505720648681745,
    "p95_s_adjusted": 14.911944263335316,
    "p95_s_raw": 7.380972131667658
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3.5-0.8B through SimpleJev native assistant-prefill option-logit scorer",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      43.866426363812614,
      53.741885017483725
     ],
     "I": [
      8.322750852470811,
      16.973486697243555
     ],
     "capability": [
      26.61188293230259,
      34.97208009895024
     ],
     "composite": null,
     "seed": 100
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 49.46821562395294,
     "capability": 31.07287020978288,
     "intelligence": 12.67752479561282
    },
    "per_type": {
     "choice": {
      "accuracy": 0.336,
      "calibration": 54.832339895302255,
      "cc": 15.0838835576859,
      "cc_by_type": {
       "choice": 15.0838835576859
      },
      "cc_tiered": 16.144822171933782,
      "cc_tiered_open": 16.101545239125173,
      "cc_tiered_sealed": 16.18809910474239,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 15.0838835576859
     },
     "noul": {
      "accuracy": 0.5733333333333334,
      "calibration": 18.523449039070215,
      "cc": 14.666666666666671,
      "cc_by_type": {
       "noul": 14.666666666666671
      },
      "cc_tiered": 20.320912200397466,
      "cc_tiered_open": 31.332643202208423,
      "cc_tiered_sealed": 9.309181198586508,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 14.666666666666671
     },
     "score": {
      "accuracy": 0.3253333333333333,
      "calibration": 75.04885793748636,
      "cc": -3.0837213034381916,
      "cc_by_type": {
       "score": -3.0837213034381916
      },
      "cc_tiered": 1.5668400145072177,
      "cc_tiered_open": 3.1336800290144353,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 51.07750814673811,
     "capability": 31.902855962425715,
     "intelligence": 12.728203778113318
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.022666666666666613,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.040000000000000036,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 1.7703531590976693,
    "Emphasis on Accuracy 60:20:20": 1.2052455663613058,
    "Emphasis on Cost 20:20:60": 2.3817516434877475,
    "Emphasis on Speed 20:60:20": 2.2471321652922893,
    "Intelligence only": 0.8150108628546057
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Featherless AI",
   "axes": {
    "calibration": 80.3,
    "cost": 35.15,
    "intelligence": 56.29,
    "speed": 76.09
   },
   "calibration": {
    "score": 80.3
   },
   "capability": 68.29,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     22.75,
     28.83
    ],
    "B": [
     22.25,
     29.49
    ],
    "C": [
     15.8,
     28.83
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)",
    "kind": "estimate",
    "usd_per_1000": 0.1451164454277286
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "SimpleJev Qwen3.6-35B-A3B",
   "endpoint_condition": "the author's public demo endpoint",
   "endpoint_kind": "demo",
   "intelligence": {
    "I_open": 60.15,
    "I_sealed": 47.27,
    "base": 56.29,
    "per_type_split": {
     "open|choice": {
      "cc": 73.3677,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.5405,
       "hard": 62.2523,
       "judge": 88.1345,
       "standard": 69.8619
      }
     },
     "open|noul": {
      "cc": 55.5473,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.7143,
       "hard": 65.2174,
       "judge": 40,
       "standard": 44.4444
      }
     },
     "open|score": {
      "cc": 51.55,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 55.6168,
       "hard": 57.237,
       "judge": 65.3065,
       "standard": 17.5079
      }
     },
     "sealed|choice": {
      "cc": 60.9118,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 83.3179,
       "hard": 41.8292,
       "judge": 68.3417,
       "standard": 76.7292
      }
     },
     "sealed|noul": {
      "cc": 40.5963,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 88.2353,
       "hard": 18.1818,
       "judge": 40,
       "standard": 62.5
      }
     },
     "sealed|score": {
      "cc": 40.3111,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 59.2125,
       "hard": 8.1459,
       "judge": 57.1294,
       "standard": 69.9633
      }
     }
    }
   },
   "jevbench_score": 27.53,
   "key": "simplejev-qwen3.6-35b-a3b",
   "last_measured_on": "2026-10-06",
   "licence": "Apache-2.0 (Qwen weights); repository licence not stated",
   "listing": "ranked",
   "method": "v1.7",
   "open": "yes",
   "previous": {
    "api_board_rank": 33,
    "axes": {
     "calibration": 80.3,
     "cost": 35.15,
     "intelligence": 56.29,
     "speed": 76.7
    },
    "board_release": "v1.6.43",
    "capability": 68.29,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      22.78,
      28.87
     ],
     "B": [
      22.27,
      29.53
     ],
     "C": [
      15.82,
      28.87
     ]
    },
    "jevbench_score": 27.57,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 106,
    "ranks": {
     "A": 106,
     "B": 100,
     "C": 95,
     "capability": 88
    },
    "scores": {
     "A": 27.57,
     "B": 27.61,
     "C": 24.26
    },
    "speed": {
     "adjustment": "x2 (demo)",
     "n": 591,
     "p50_s_adjusted": 1.25,
     "p50_s_raw": 0.625,
     "p95_s_adjusted": 1.709,
     "p95_s_raw": 0.8545
    },
    "views": null
   },
   "rank": 171,
   "ranked": true,
   "ranks": {
    "A": 171,
    "B": 161,
    "C": 158,
    "capability": 114
   },
   "repo": "https://github.com/featherless-ai/simple-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 27.53,
    "B": 27.58,
    "C": 24.22
   },
   "speed": {
    "adjustment": "x2 (demo)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.304,
    "p50_s_raw": 0.652,
    "p95_s_adjusted": 1.885,
    "p95_s_raw": 0.943
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "7cd32be5f5c8177492a086c471436c956ff89527c8b26c3705c4e939597ffd6e"
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Featherless AI",
   "axes": {
    "calibration": 90.33,
    "cost": 14.9,
    "intelligence": 52.88,
    "speed": 76.44
   },
   "calibration": {
    "score": 90.33
   },
   "capability": 71.6,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     2.1,
     3.31
    ],
    "B": [
     2.18,
     3.6
    ],
    "C": [
     1.46,
     3.31
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: exact base-model market reference (operator tariff excluded by 30-day rule)",
    "kind": "estimate",
    "usd_per_1000": 0.686803448275862
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "SimpleJev Qwen3.8-27B",
   "endpoint_kind": "demo",
   "intelligence": {
    "I_open": 58.03,
    "I_sealed": 42.58,
    "base": 52.88,
    "per_type_split": {
     "open|choice": {
      "cc": 77.1205,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.2162,
       "hard": 67.9144,
       "judge": 84.1793,
       "standard": 77.3964
      }
     },
     "open|noul": {
      "cc": 39.7633,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 39.1304,
       "judge": 10,
       "standard": 55.5556
      }
     },
     "open|score": {
      "cc": 57.2101,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 75.6453,
       "hard": 49.2044,
       "judge": 65.6588,
       "standard": 51.3309
      }
     },
     "sealed|choice": {
      "cc": 61.5454,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 72.1965,
       "hard": 47.6462,
       "judge": 63.8191,
       "standard": 80.6076
      }
     },
     "sealed|noul": {
      "cc": 31.4198,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 76.4706,
       "hard": 18.1818,
       "judge": 30,
       "standard": 37.5
      }
     },
     "sealed|score": {
      "cc": 34.7637,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 72.2528,
       "hard": 8.1052,
       "judge": 33.119,
       "standard": 71.8031
      }
     }
    }
   },
   "jevbench_score": 3.22,
   "key": "simplejev-qwen3.8-27b",
   "last_measured_on": "2026-10-06",
   "licence": "Apache-2.0 (Qwen weights); repository licence not stated",
   "listing": "ranked",
   "method": "v1.7",
   "open": "yes",
   "previous": {
    "api_board_rank": 41,
    "axes": {
     "calibration": 90.33,
     "cost": 14.9,
     "intelligence": 52.88,
     "speed": 76.93
    },
    "board_release": "v1.6.43",
    "capability": 71.6,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      2.11,
      3.31
     ],
     "B": [
      2.18,
      3.6
     ],
     "C": [
      1.46,
      3.31
     ]
    },
    "jevbench_score": 3.22,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 193,
    "ranks": {
     "A": 193,
     "B": 186,
     "C": 190,
     "capability": 66
    },
    "scores": {
     "A": 3.22,
     "B": 3.44,
     "C": 2.5
    },
    "speed": {
     "adjustment": "x2 (demo)",
     "n": 591,
     "p50_s_adjusted": 1.212,
     "p50_s_raw": 0.606,
     "p95_s_adjusted": 1.675,
     "p95_s_raw": 0.8375
    },
    "views": null
   },
   "rank": 234,
   "ranked": true,
   "ranks": {
    "A": 234,
    "B": 228,
    "C": 233,
    "capability": 83
   },
   "repo": "https://github.com/featherless-ai/simple-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.22,
    "B": 3.44,
    "C": 2.5
   },
   "speed": {
    "adjustment": "x2 (demo)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.266,
    "p50_s_raw": 0.633,
    "p95_s_adjusted": 1.794,
    "p95_s_raw": 0.897
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "f12f4c57ca739705117c34ceb3f847b59b15f9744120c83faf519a83ba9c123b"
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Featherless",
   "axes": {
    "calibration": 91.85610710600702,
    "cost": 42.47169570460766,
    "intelligence": 57.92931033189586,
    "speed": 75.96701882114468
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0559935150394999,
      "ece_score": 88.80129699210002,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.80129699210002,
      "score_before_time_limit": 88.80129699210002,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.21832884097035044,
      "brier": 0.07396200744195928,
      "ece": 0.02357869016917279,
      "n": 371,
      "score": 95.28426196616545,
      "score_before_time_limit": 95.28426196616545,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9701897018970189,
      "interval90_width": 0.2899728997289972,
      "n": 369,
      "nrps": 0.061950920450199856,
      "score": 91.48276235975561,
      "score_before_time_limit": 91.48276235975561,
      "time_limit_failed_share": 0,
      "top_ece": 0.05419691617734392
     }
    },
    "score": 91.85610710600702
   },
   "capability": 74.89270871895144,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,377,206 input + 0 output tokens over 1,477 decisions = 1609.48273527 tokens/decision. GPU-time cost USD 0.0827199271319624/1,000; old list-price cost USD 0.675982748815166/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0827199271319624/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.66561768
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.08271992713196244
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SimpleJev Qwen3.8-27B (self-hosted, v1.6 pool)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 58.77016996644363,
    "I_sealed": 57.08845069734809,
    "base": 57.92931033189586,
    "excess": -0.8916683733426987,
    "gap": 1.6817192690955451,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 79.38232847559632,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 69.80182447310474,
       "judge": 84.17930125247197,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 39.7632850241546,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 57.16489639957998,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.0815846586252,
       "hard": 50.99462651293531,
       "judge": 64.6276543190174,
       "standard": 50.35295516419056
      }
     },
     "sealed|choice": {
      "cc": 67.17049093981976,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 61.312217194570124,
       "judge": 64.61824953445065,
       "standard": 78.93829289321346
      }
     },
     "sealed|noul": {
      "cc": 46.31434719713273,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": 26.21359223300972,
       "judge": 52.63157894736843,
       "standard": 66.19718309859155
      }
     },
     "sealed|score": {
      "cc": 57.78051395509178,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 70.9707943343972,
       "hard": 48.80809948321188,
       "judge": 65.15163618443506,
       "standard": 58.0735193651839
      }
     }
    }
   },
   "jevbench_score": 44.49974984123803,
   "key": "simplejev-qwen3.8-27b-selfhost",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0 + featherless-ai/simple-jev@9c11582d3e631e0b118f3f3132ebbb00c62020c5",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9620689655172414,
    "decisive_rate": 0.7816711590296496,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.85610710600702,
     "cost": 15.10193161538902,
     "intelligence": 57.92931033189586,
     "speed": 76.19964460590472
    },
    "board_release": "v1.6.43",
    "capability": 74.89270871895144,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      3.3246680487142375,
      3.4292430519298938
     ],
     "B": [
      3.5406927144972373,
      3.7245951371091817
     ],
     "C": [
      2.5475929721327564,
      3.4292430519298938
     ]
    },
    "jevbench_score": 3.394848880196433,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 149,
    "rank": 189,
    "ranks": {
     "A": 189,
     "B": 185,
     "C": 185,
     "capability": 45
    },
    "scores": {
     "A": 3.394848880196433,
     "B": 3.656360936982179,
     "C": 3.164569675341542
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.393547774007311,
     "p50_s_raw": 0.6217738870036555,
     "p95_s_adjusted": 1.721526361943688,
     "p95_s_raw": 0.785763180971844
    },
    "views": {
     "Balanced 33:33:33": 2.833072735614859,
     "Emphasis on Accuracy 60:20:20": 3.4785771436394777,
     "Emphasis on Cost 20:20:60": 1.9915507939259591,
     "Emphasis on Speed 20:60:20": 3.7129751262531117,
     "Intelligence only": 5.284736623506664
    }
   },
   "provenance": {
    "carried_metadata_from": "author repository + base config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 132,
   "ranked": true,
   "ranks": {
    "A": 132,
    "B": 125,
    "C": 115,
    "capability": 64
   },
   "repo": "https://github.com/featherless-ai/simple-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 44.49974984123803,
    "B": 43.93185823857032,
    "C": 41.48124522695047
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.42801156102214,
    "p50_s_raw": 0.63900578051107,
    "p95_s_adjusted": 1.7724189897620815,
    "p95_s_raw": 0.8112094948810408
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.56600011746222,
      91.91458842585743
     ],
     "I": [
      52.49374537062954,
      62.50608217507831
     ],
     "capability": [
      70.32359564579012,
      76.5602273459949
     ],
     "composite": null,
     "seed": 112
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.85610710600702,
     "capability": 74.89270871895144,
     "intelligence": 57.92931033189586
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7653333333333333,
      "calibration": 88.80129699210002,
      "cc": 69.98948495211388,
      "cc_by_type": {
       "choice": 69.98948495211388
      },
      "cc_tiered": 73.27640970770804,
      "cc_tiered_open": 79.38232847559632,
      "cc_tiered_sealed": 67.17049093981976,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.98948495211388
     },
     "noul": {
      "accuracy": 0.744,
      "calibration": 95.28426196616545,
      "cc": 48.8,
      "cc_by_type": {
       "noul": 48.8
      },
      "cc_tiered": 43.038816110643666,
      "cc_tiered_open": 39.7632850241546,
      "cc_tiered_sealed": 46.31434719713273,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 48.8
     },
     "score": {
      "accuracy": 0.6906666666666667,
      "calibration": 91.48276235975561,
      "cc": 57.30970745563964,
      "cc_by_type": {
       "score": 57.30970745563964
      },
      "cc_tiered": 57.47270517733588,
      "cc_tiered_open": 57.16489639957998,
      "cc_tiered_sealed": 57.78051395509178,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 57.30970745563964
     }
    },
    "raw_on_A": {
     "calibration": 90.15198763991802,
     "capability": 76.03520978092624,
     "intelligence": 61.91843192193445
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 40.10689412984208,
    "Emphasis on Accuracy 60:20:20": 40.766720491243625,
    "Emphasis on Cost 20:20:60": 35.69803441048148,
    "Emphasis on Speed 20:60:20": 44.92857760492121,
    "Intelligence only": 41.798197235839524
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Singularity Compute",
   "axes": {
    "calibration": 90.84828412834014,
    "cost": 64.26424619973979,
    "intelligence": 78.8866847061141,
    "speed": 85.30638666270315
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05436905982497981,
      "ece_score": 89.12618803500403,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 88.413178530724,
      "score_before_time_limit": 89.12618803500403,
      "time_limit_failed_share": 0.008
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.058022045174915465,
      "ece": 0.04121594125547194,
      "n": 375,
      "score": 91.26744208624478,
      "score_before_time_limit": 91.7568117489056,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.92,
      "interval90_width": 0.10022222222222223,
      "n": 375,
      "nrps": 0.04473236689969343,
      "score": 92.86423176805167,
      "score_before_time_limit": 93.61313686295532,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.041502447920600115
     }
    },
    "score": 90.84828412834014
   },
   "capability": 84.86748441722712,
   "composite_ci95": {
    "A": [
     76.93989597679156,
     79.39169870780026
    ],
    "B": [
     76.64350103296658,
     79.9065768646065
    ],
    "C": [
     76.93989597679156,
     79.39169870780026
    ]
   },
   "cost": {
    "basis": "Launch list price: the launch list price stated on the vendor's public product page (singularitycompute.com/super-decider, fetched 22 Oct 2026: USD 0.03 per 1M input tokens, output tokens free); the ASI:Cloud catalogue itself was not reachable to us. The page lists one price for the product and no separate rate for thinking mode; we apply it to both rows. Counted as an operator launch list price by the frozen pricing module pricing_v15 (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen on 22 Oct 2026 with launch-list keys added and nothing else changed; the launch-list line and the rates for this product are identical in the earlier module 81e3578f), which carries a launch-list-price line for this product. Applied to the tokens reported by the provider's API; not a base-model estimate. Over all 1,500 items the frozen resolve() gives USD 0.039197 per 1,000 decisions; the official cost axis uses the common 1,485-item basis, where it is USD 0.015531 per 1,000 decisions. COMMON COST BASIS (v1.6.30): common 1,485 items, excluding the 15 longest for every row; 768,771 input / 322,546 output tokens. All-item usage 1,959,826 input / 328,348 output over 1,500 items.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.03919652
    },
    "kind": "listed",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.01553072727272727
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 11,
    "n_over_limit_answered_ok": 11,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Super Decider",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid fast-lane evaluation, published at the provider's request (the order was private and was made public at the customer's request). Vendor-hosted, first-party Singularity Compute API (llm.a.singularitycompute.com, POST /v1/systemone, model `super-decider`), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `singularitycompute` (no proxy) before the first request. All 1,500 of 1,500 requests answered with HTTP 200 and the served model was `super-decider` on every row; native probabilities on all three question types; one serial pass, no retries. Run with the provider's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 78.80985381401513,
    "I_sealed": 78.96351559821305,
    "base": 78.8866847061141,
    "excess": -9.993188554553633,
    "gap": -0.1536617841979222,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 82.86017015722445,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 77.35136835482857,
       "judge": 88.13447593935399,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 84.04347826086956,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 70,
       "standard": 100
      }
     },
     "open|score": {
      "cc": 69.5259130239514,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 85.38361656115687,
       "hard": 81.92486849848484,
       "judge": 59.862361248832244,
       "standard": 51.294477968960514
      }
     },
     "sealed|choice": {
      "cc": 81.34124143256045,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 75.02023823291314,
       "judge": 80.60724779627816,
       "standard": 87.66083104830204
      }
     },
     "sealed|noul": {
      "cc": 83.99136221472814,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 78.18181818181817,
       "judge": 84.61538461538463,
       "standard": 97.01492537313432
      }
     },
     "sealed|score": {
      "cc": 71.55794314735057,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 88.87375089665288,
       "hard": 62.94798902769401,
       "judge": 75.64215151608957,
       "standard": 73.99363495890405
      }
     }
    }
   },
   "jevbench_score": 78.48219079083019,
   "key": "singularity-super-decider",
   "last_measured_on": "2026-10-21",
   "licence": "Apache-2.0 (Hugging Face model card; weights public, not gated); the card states that the licence is the same as that of the base model google/diffusiongemma-26B-A4B-it; the hosted service itself is proprietary",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:llm.a.singularitycompute.com/v1/systemone#super-decider",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9276139410187667,
    "decisive": 373,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "rank": 2,
   "ranked": true,
   "ranks": {
    "A": 2,
    "B": 1,
    "C": 2,
    "capability": 10
   },
   "repo": "https://singularitycompute.com/super-decider",
   "scores": {
    "A": 78.48219079083019,
    "B": 78.56275738441518,
    "C": 78.48219079083019
   },
   "source_url": "https://singularitycompute.com/super-decider",
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.24612953246105462,
    "p50_s_raw": 0.24612953246105462,
    "p95_s_adjusted": 1.1972851804515812,
    "p95_s_raw": 1.1972851804515812
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API: no vendor code ran locally. Model, as pinned by the provider: Singularity-Compute/super-decider at 7f8b3f76e88aba6aa1159da75eb577ea990d2af1 on Hugging Face (public, not gated), a discrete-diffusion decision model fine-tuned from google/diffusiongemma-26B-A4B-it and served from an NVFP4 checkpoint (card statement). Serving code, as pinned by the provider: github.com/Singularity-Compute/super-decider at 10bf7eddcbd3b5f6dddf27e868231b2e1140dfba (pinned vLLM nightly with the provider's patches and a structured server). Neither the deployed weights nor the serving stack were independently inspected. MANDATORY DISCLOSURE: measured through the provider's hosted API on 21 Oct 2026; published at the provider's request. THINKING MODE: the API default (no `thinking` field, i.e. \"auto\"); the row shows one fixed setting, and the provider's docs say that in \"auto\" mode thinking is skipped when the server is busy, so the default row reflects the server load at the time of measurement (21 Oct 2026, 08:05-08:45 UTC). TRAINING AND SELECTION, provider statement: the public JevBench v1.4/v1.5 items were used as a validation set (recipe and checkpoint choice, serving parameters), not as training data, and the model card says no benchmark test items were used for training; the public set is therefore treated as seen during selection and the normal public-versus-sealed gap rule applies to this row. Provider statements are not independently verified.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 89.85545147257876,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 69.56635441773629
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 89.85545147257876
      },
      "calibration_gen": {
       "accuracy": 0.9702970297029703,
       "cc": 94.85874669366245,
       "cc_by_type": {
        "choice": 96.8186638388123,
        "noul": 92,
        "score": 95.75757624217506
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 94.85874669366245
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 85.67625328152366,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 77.77777777777777,
        "score": 95.5111446684192
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 85.67625328152366
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.12234530239441,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.36703590718322
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.12234530239441
      },
      "jaggedness": {
       "accuracy": 0.8722222222222222,
       "cc": 82.77262796695852,
       "cc_by_type": {
        "choice": 79.78825794032724,
        "noul": 94.59459459459461,
        "score": 73.93503136595372
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 82.77262796695852
      },
      "llm_router": {
       "accuracy": 0.8817891373801917,
       "cc": 78.90312865239771,
       "cc_by_type": {
        "choice": 82.56581843796776,
        "noul": 97.93814432989691,
        "score": 56.205423189328485
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 78.90312865239771
      },
      "long_state": {
       "accuracy": 0.13333333333333333,
       "cc": -134.40860215053763,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.8888888888888888,
       "cc": 82.2008021480952,
       "cc_by_type": {
        "choice": 87.62886597938144,
        "noul": 100,
        "score": 58.973540464904175
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 82.2008021480952
      },
      "multilingual": {
       "accuracy": 0.8436363636363636,
       "cc": 76.30071025864956,
       "cc_by_type": {
        "choice": 78.4805999347897,
        "noul": 75,
        "score": 75.42153084115898
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 76.30071025864956
      },
      "negation_pair_gen": {
       "accuracy": 0.8837209302325582,
       "cc": 74.63974664616975,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 60.00000000000001,
        "score": 70.58590660517592
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.63974664616975
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.9375,
       "cc": 94.1393156079887,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 94.42995883597814
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 94.1393156079887
      },
      "usecase": {
       "accuracy": 0.7636363636363637,
       "cc": 69.35577915701994,
       "cc_by_type": {
        "choice": 70.58823529411767,
        "noul": 75,
        "score": 62.47910217694213
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 69.35577915701994
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 81.19799542871031,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": 62.64160533375001
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 81.19799542871031
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 79.87151540781498,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 100,
        "score": 60.35187341238502
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 79.87151540781498
      },
      "de": {
       "accuracy": 0.875,
       "cc": 85.97225175792362,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 81.81056058350534
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 85.97225175792362
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 65.82742536986764,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 97.6684958488953
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 65.82742536986764
      },
      "en": {
       "accuracy": 0.8757602085143353,
       "cc": 80.5020795009644,
       "cc_by_type": {
        "choice": 82.86572045740228,
        "noul": 85.61151079136691,
        "score": 73.02900725412404
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 80.5020795009644
      },
      "es": {
       "accuracy": 0.9354838709677419,
       "cc": 86.76999083304393,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 70.09790072099149
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 86.76999083304393
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 64.27731458013723,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 92.28549565298
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 64.27731458013723
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 83.31529512153791,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 71.12235595284902
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 83.31529512153791
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 72.25647563538706,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100,
        "score": 16.76942690616119
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 72.25647563538706
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 76.06266465412442,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 42.217612044992116
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 76.06266465412442
      },
      "ja": {
       "accuracy": 0.88,
       "cc": 80.36396885359544,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 100,
        "score": 69.68602888008735
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 80.36396885359544
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 59.1851949497821,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 77.55558484934632
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 59.1851949497821
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.42623540510623,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.27870621531868
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.42623540510623
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 45.29308709163208,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 54.49397988961484
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 45.29308709163208
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 75,
       "cc_by_type": {
        "choice": 100,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 75
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 76.21121333785727,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80,
        "score": 56.25107547696585
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 76.21121333785727
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 77.45853245441235,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 74.80341115227651
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 77.45853245441235
      },
      "sv": {
       "accuracy": 0.8181818181818182,
       "cc": 62.003275945801796,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 26.009827837405396
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 62.003275945801796
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 84.17252034238425,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 74.25669146193536
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 84.17252034238425
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 68.25214444086528,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 69.27256235485393
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 68.25214444086528
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 67.66972367558195,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": 95.09956650697184
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 67.66972367558195
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8626666666666667,
       "cc": 82.26142865343769,
       "cc_by_type": {
        "choice": 82.26142865343769
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 82.26142865343769
      },
      "noul": {
       "accuracy": 0.9226666666666666,
       "cc": 84.53333333333333,
       "cc_by_type": {
        "noul": 84.53333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 84.53333333333333
      },
      "score": {
       "accuracy": 0.8293333333333334,
       "cc": 72.07798903636777,
       "cc_by_type": {
        "score": 72.07798903636777
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 72.07798903636777
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 27.939831762844438,
       "cc_by_type": {
        "choice": 84,
        "noul": -25,
        "score": 24.81949528853331
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 27.939831762844438
      },
      "code_linting": {
       "accuracy": 0.8333333333333334,
       "cc": 77.33853334905363,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 50.64914041983174
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 77.33853334905363
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 82.441425967922,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100,
        "score": 96.10476570864405
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 82.441425967922
      },
      "demand_forecasting": {
       "accuracy": 0.5882352941176471,
       "cc": 60.55897120398711,
       "cc_by_type": {
        "choice": 41.5204678362573,
        "noul": 100,
        "score": 40.15644577570404
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 60.55897120398711
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8823529411764706,
       "cc": 68.03216403821983,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 70.76315878132617
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 68.03216403821983
      },
      "feature_extraction": {
       "accuracy": 0.8947368421052632,
       "cc": 54.12246561727883,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 66.66666666666667,
        "score": 8.279346537371046
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 54.12246561727883
      },
      "financial_crime": {
       "accuracy": 0.7333333333333333,
       "cc": 61.89527890054794,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 42.85714285714286,
        "score": 52.632615413128406
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 61.89527890054794
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 65.03231166234393,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": 50,
        "score": 76.45582000445339
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 65.03231166234393
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 69.68099890061212,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 100,
        "score": 92.3763300351697
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 69.68099890061212
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 65.94889455125153,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 100,
        "score": 36.390580870028685
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 65.94889455125153
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 65.62556455092009,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 77.77777777777777,
        "score": 42.35472982847086
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 65.62556455092009
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 62.45738322463984,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100,
        "score": 84.14634322230664
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 62.45738322463984
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 68.17118595744371,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 60.00000000000001,
        "score": 67.54618743087235
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 68.17118595744371
      },
      "model_routing": {
       "accuracy": 0.8847926267281107,
       "cc": 77.08049546993229,
       "cc_by_type": {
        "choice": 88.31168831168831,
        "noul": 96.96969696969697,
        "score": 45.960101128411566
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 77.08049546993229
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 80.32276037340623,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100,
        "score": 64.00091067875992
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 80.32276037340623
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 55.50483084561898,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 66.60734304753477
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 55.50483084561898
      },
      "risk_assessment": {
       "accuracy": 0.8333333333333334,
       "cc": 78.24962814912611,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 100,
        "score": 64.30553469368374
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 78.24962814912611
      },
      "scientific_discovery": {
       "accuracy": 0.8260869565217391,
       "cc": 81.73460160704407,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 100,
        "score": 74.18931206750902
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 81.73460160704407
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 94.63219304596917,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 94.5727357215374
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 94.63219304596917
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.86973938739871,
      92.5387564621637
     ],
     "I": [
      75.18214623594949,
      82.12723110306345
     ],
     "capability": [
      81.40391281836617,
      86.85759269246338
     ],
     "composite": {
      "A": [
       76.93989597679156,
       79.39169870780026
      ],
      "B": [
       76.64350103296658,
       79.9065768646065
      ],
      "C": [
       76.93989597679156,
       79.39169870780026
      ]
     },
     "seed": 75
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.84828412834014,
     "capability": 84.86748441722712,
     "intelligence": 78.8866847061141
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8626666666666667,
      "calibration": 88.413178530724,
      "cc": 82.26142865343769,
      "cc_by_type": {
       "choice": 82.26142865343769
      },
      "cc_tiered": 82.10070579489245,
      "cc_tiered_open": 82.86017015722445,
      "cc_tiered_sealed": 81.34124143256045,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 82.26142865343769
     },
     "noul": {
      "accuracy": 0.9226666666666666,
      "calibration": 91.26744208624478,
      "cc": 84.53333333333333,
      "cc_by_type": {
       "noul": 84.53333333333333
      },
      "cc_tiered": 84.01742023779886,
      "cc_tiered_open": 84.04347826086956,
      "cc_tiered_sealed": 83.99136221472814,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 84.53333333333333
     },
     "score": {
      "accuracy": 0.8293333333333334,
      "calibration": 92.86423176805167,
      "cc": 72.07798903636777,
      "cc_by_type": {
       "score": 72.07798903636777
      },
      "cc_tiered": 70.541928085651,
      "cc_tiered_open": 69.5259130239514,
      "cc_tiered_sealed": 71.55794314735057,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 72.07798903636777
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.008000000000000007,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 75.07579973698547,
    "Emphasis on Accuracy 60:20:20": 76.55509972242248,
    "Emphasis on Cost 20:20:60": 70.34216842661972,
    "Emphasis on Speed 20:60:20": 78.85873400314371,
    "Intelligence only": 78.8866847061141
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Singularity Compute",
   "axes": {
    "calibration": 91.69553288708067,
    "cost": 56.397623391206686,
    "intelligence": 80.00899811568502,
    "speed": 79.83008200555805
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.035608016166795625,
      "ece_score": 92.87839676664088,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 92.63072104192983,
      "score_before_time_limit": 92.87839676664088,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.05523784944949169,
      "ece": 0.03925753215871338,
      "n": 375,
      "score": 91.9027642520753,
      "score_before_time_limit": 92.14849356825732,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.9226666666666666,
      "interval90_width": 0.10066666666666665,
      "n": 375,
      "nrps": 0.046295265602298664,
      "score": 90.5531133672369,
      "score_before_time_limit": 90.79523399121348,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.06890002728671593
     }
    },
    "score": 91.69553288708067
   },
   "capability": 85.85226550138285,
   "composite_ci95": {
    "A": [
     73.29668924375432,
     75.30976141875034
    ],
    "B": [
     73.94546816229918,
     76.78676004590665
    ],
    "C": [
     73.29668924375432,
     75.30976141875034
    ]
   },
   "cost": {
    "basis": "Launch list price: the launch list price stated on the vendor's public product page (singularitycompute.com/super-decider, fetched 22 Oct 2026: USD 0.03 per 1M input tokens, output tokens free); the ASI:Cloud catalogue itself was not reachable to us. The page lists one price for the product and no separate rate for thinking mode; we apply it to both rows. Counted as an operator launch list price by the frozen pricing module pricing_v15 (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen on 22 Oct 2026 with launch-list keys added and nothing else changed; the launch-list line and the rates for this product are identical in the earlier module 81e3578f), which carries a launch-list-price line for this product. Applied to the tokens reported by the provider's API; not a base-model estimate. Over all 1,500 items the frozen resolve() gives USD 0.052098 per 1,000 decisions; the official cost axis uses the common 1,485-item basis, where it is USD 0.028406 per 1,000 decisions. COMMON COST BASIS (v1.6.30): common 1,485 items, excluding the 15 longest for every row; 1,406,105 input / 636,221 output tokens. All-item usage 2,604,915 input / 643,964 output over 1,500 items.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05209830000000001
    },
    "kind": "listed",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.028406161616161614
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 4,
    "n_over_limit_answered_ok": 4,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Super Decider (thinking)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-21",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Paid fast-lane evaluation, published at the provider's request (the order was private and was made public at the customer's request). Vendor-hosted, first-party Singularity Compute API (llm.a.singularitycompute.com, POST /v1/systemone, model `super-decider`), through the unchanged TypeSafe adapter; latency measured from Sandy (Germany) without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `singularitycompute` (no proxy) before the first request. All 1,500 of 1,500 requests answered with HTTP 200 and the served model was `super-decider` on every row; native probabilities on all three question types; one serial pass, no retries. Run with the provider's own API key; hosting implementation and deployed weights were not independently inspected.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 78.9137241571549,
    "I_sealed": 81.10427207421513,
    "base": 80.00899811568502,
    "excess": -12.03007468741594,
    "gap": -2.190547917060229,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 89.43664280726739,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 81.12614029569046,
       "judge": 100,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 80.56521739130434,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 70,
       "standard": 100
      }
     },
     "open|score": {
      "cc": 66.73931227289297,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 87.8266479316451,
       "hard": 83.83079175879907,
       "judge": 53.50151232669309,
       "standard": 41.86938539100456
      }
     },
     "sealed|choice": {
      "cc": 86.07206252990329,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 79.18353186076095,
       "judge": 87.65915768854063,
       "standard": 91.45749841805527
      }
     },
     "sealed|noul": {
      "cc": 85.33314138254951,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 78.18181818181817,
       "judge": 90.76923076923077,
       "standard": 91.04477611940298
      }
     },
     "sealed|score": {
      "cc": 71.9076123101926,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 88.61860278662654,
       "hard": 70.62401785520835,
       "judge": 70.00494391151904,
       "standard": 68.97330857995443
      }
     }
    }
   },
   "jevbench_score": 74.54050341581339,
   "key": "singularity-super-decider-thinking",
   "last_measured_on": "2026-10-21",
   "licence": "Apache-2.0 (Hugging Face model card; weights public, not gated); the card states that the licence is the same as that of the base model google/diffusiongemma-26B-A4B-it; the hosted service itself is proprietary",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:llm.a.singularitycompute.com/v1/systemone#super-decider",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9278074866310161,
    "decisive": 374,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "rank": 25,
   "ranked": true,
   "ranks": {
    "A": 25,
    "B": 9,
    "C": 22,
    "capability": 6
   },
   "repo": "https://singularitycompute.com/super-decider",
   "scores": {
    "A": 74.54050341581339,
    "B": 75.57357140671725,
    "C": 74.54050341581339
   },
   "source_url": "https://singularitycompute.com/super-decider",
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.8880362859927118,
    "p50_s_raw": 0.8880362859927118,
    "p95_s_adjusted": 1.171011304546846,
    "p95_s_raw": 1.171011304546846
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Vendor-hosted API: no vendor code ran locally. Model, as pinned by the provider: Singularity-Compute/super-decider at 7f8b3f76e88aba6aa1159da75eb577ea990d2af1 on Hugging Face (public, not gated), a discrete-diffusion decision model fine-tuned from google/diffusiongemma-26B-A4B-it and served from an NVFP4 checkpoint (card statement). Serving code, as pinned by the provider: github.com/Singularity-Compute/super-decider at 10bf7eddcbd3b5f6dddf27e868231b2e1140dfba (pinned vLLM nightly with the provider's patches and a structured server). Neither the deployed weights nor the serving stack were independently inspected. MANDATORY DISCLOSURE: measured through the provider's hosted API on 21 Oct 2026; published at the provider's request. THINKING MODE: `\"thinking\": \"always\"` in every request body; the row shows one fixed setting, and the provider's docs say that in \"auto\" mode thinking is skipped when the server is busy, so the default row reflects the server load at the time of measurement (21 Oct 2026, 08:05-08:45 UTC). TRAINING AND SELECTION, provider statement: the public JevBench v1.4/v1.5 items were used as a validation set (recipe and checkpoint choice, serving parameters), not as training data, and the model card says no benchmark test items were used for training; the public set is therefore treated as seen during selection and the normal public-versus-sealed gap rule applies to this row. Provider statements are not independently verified.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 72.35548687610432,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 67.06646062831297
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 72.35548687610432
      },
      "calibration_gen": {
       "accuracy": 0.9603960396039604,
       "cc": 94.04767911079934,
       "cc_by_type": {
        "choice": 96.8186638388123,
        "noul": 92,
        "score": 93.32437349358574
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 94.04767911079934
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 81.46444618137986,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 70.37037037037037,
        "score": 90.2831307753952
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 81.46444618137986
      },
      "fairness": {
       "accuracy": 0.9791666666666666,
       "cc": 94.69056759200323,
       "cc_by_type": {
        "choice": 100,
        "noul": 86.66666666666667,
        "score": 97.40503610934303
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 94.69056759200323
      },
      "jaggedness": {
       "accuracy": 0.9277777777777778,
       "cc": 86.73264775638553,
       "cc_by_type": {
        "choice": 91.33782483156881,
        "noul": 94.59459459459461,
        "score": 74.26552384299319
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 86.73264775638553
      },
      "llm_router": {
       "accuracy": 0.8913738019169329,
       "cc": 79.28475475311417,
       "cc_by_type": {
        "choice": 88.1130580258871,
        "noul": 97.93814432989691,
        "score": 51.8030619035585
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 79.28475475311417
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": -27.822335394381014,
       "cc_by_type": {
        "choice": 22.58064516129032,
        "noul": -100,
        "score": -6.047651344433369
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.8888888888888888,
       "cc": 83.5588517898443,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 100,
        "score": 75.41882341076999
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 83.5588517898443
      },
      "multilingual": {
       "accuracy": 0.8727272727272727,
       "cc": 77.97925523642682,
       "cc_by_type": {
        "choice": 87.28399087055756,
        "noul": 75,
        "score": 71.65377483872291
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 77.97925523642682
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 81.25264963526998,
       "cc_by_type": {
        "choice": 90,
        "noul": 73.33333333333334,
        "score": 80.42461557247661
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 81.25264963526998
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.984375,
       "cc": 97.86154958999055,
       "cc_by_type": {
        "choice": 96.996996996997,
        "noul": 100,
        "score": 96.58765177297464
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 97.86154958999055
      },
      "usecase": {
       "accuracy": 0.7863636363636364,
       "cc": 72.69446270220719,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 83.33333333333333,
        "score": 61.71083908701371
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 72.69446270220719
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8125,
       "cc": 81.37414726403654,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100,
        "score": 82.21767988734771
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 81.37414726403654
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 86.74302168543532,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 60.22906505630594
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 86.74302168543532
      },
      "de": {
       "accuracy": 0.90625,
       "cc": 87.86469329901563,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 100,
        "score": 79.52328343686987
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 87.86469329901563
      },
      "el": {
       "accuracy": 0.8666666666666667,
       "cc": 82.77739181238248,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 66.66666666666667,
        "score": 98.42528530679364
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 82.77739181238248
      },
      "en": {
       "accuracy": 0.893136403127715,
       "cc": 82.44535092865597,
       "cc_by_type": {
        "choice": 87.03945521777865,
        "noul": 86.33093525179856,
        "score": 73.9656623163907
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 82.44535092865597
      },
      "es": {
       "accuracy": 0.9032258064516129,
       "cc": 80.77921693565571,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 52.12557902882683
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 80.77921693565571
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 87.0450308763467,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": 93.92197787494177
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 87.0450308763467
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 81.41948601012723,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 44.258458030381696
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 81.41948601012723
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 67.99732773933682,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100,
        "score": 3.9919832180104753
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 67.99732773933682
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 73.99076354729904,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 36.001908724516
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 73.99076354729904
      },
      "ja": {
       "accuracy": 0.88,
       "cc": 78.37958776791658,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 80,
        "score": 69.43582446340027
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 78.37958776791658
      },
      "ko": {
       "accuracy": 0.9,
       "cc": 61.87246167528537,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 85.61738502585612
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 61.87246167528537
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.5452368885293,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.63571066558788
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.5452368885293
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 52.44326408683357,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 33.33333333333333,
        "score": 49.970484901193366
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 52.44326408683357
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 75,
       "cc_by_type": {
        "choice": 100,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 75
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 67.40750200911383,
       "cc_by_type": {
        "choice": 100,
        "noul": 39.99999999999999,
        "score": 62.222506027341474
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 67.40750200911383
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 80.38189941531125,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 72.96655858771337
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 80.38189941531125
      },
      "sv": {
       "accuracy": 0.9090909090909091,
       "cc": 74.58559564417607,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 23.756786932528208
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 74.58559564417607
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 84.80472311114137,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 54.41416933342412
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 84.80472311114137
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 78.77274224161705,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 68.5762912409802
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 78.77274224161705
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 68.01735999950307,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": 96.14247547873522
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 68.01735999950307
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8986666666666666,
       "cc": 86.91134541418701,
       "cc_by_type": {
        "choice": 86.91134541418701
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 86.91134541418701
      },
      "noul": {
       "accuracy": 0.9253333333333333,
       "cc": 85.06666666666666,
       "cc_by_type": {
        "noul": 85.06666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 85.06666666666666
      },
      "score": {
       "accuracy": 0.8266666666666667,
       "cc": 72.227609837006,
       "cc_by_type": {
        "score": 72.227609837006
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 72.227609837006
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 30.4204600919081,
       "cc_by_type": {
        "choice": 68,
        "noul": 0,
        "score": 23.261380275724306
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 30.4204600919081
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 86.52558672096914,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 59.576760162907405
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 86.52558672096914
      },
      "customer_support": {
       "accuracy": 0.9285714285714286,
       "cc": 88.7634187455937,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 100,
        "score": 82.55041883840713
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 88.7634187455937
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 52.75207268586704,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 100,
        "score": 28.43165665409235
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 52.75207268586704
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8823529411764706,
       "cc": 86.10915391772893,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 100,
        "score": 72.82021537637524
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 86.10915391772893
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": -4.852470451788558,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 100,
        "score": -201.9787950031644
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 53.52947921875822,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 14.28571428571428,
        "score": 56.106644939187824
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 53.52947921875822
      },
      "gaming": {
       "accuracy": 0.9090909090909091,
       "cc": 88.761530265053,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 100,
        "score": 76.73755246763282
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 88.761530265053
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 67.58227437011047,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 100,
        "score": 86.08015644366473
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 67.58227437011047
      },
      "knowledge_graphs": {
       "accuracy": 0.7272727272727273,
       "cc": 65.89225185440681,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 66.66666666666667,
        "score": 56.70602037406981
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 65.89225185440681
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 67.00979355306558,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 23.251602881418965
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.00979355306558
      },
      "legal_compliance": {
       "accuracy": 0.7,
       "cc": 61.35217022284606,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100,
        "score": 80.8307042169253
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 61.35217022284606
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 80.38344987923516,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100,
        "score": 64.18297919624672
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 80.38344987923516
      },
      "model_routing": {
       "accuracy": 0.8663594470046083,
       "cc": 75.8636160703572,
       "cc_by_type": {
        "choice": 87.14285714285714,
        "noul": 100,
        "score": 40.44799106821445
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.8636160703572
      },
      "moderation_trust_safety": {
       "accuracy": 0.8888888888888888,
       "cc": 88.8673569663905,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 100,
        "score": 78.11838567844211
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 88.8673569663905
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 54.1509340689137,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 62.54565271741892
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.1509340689137
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 69.40657921763047,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 100,
        "score": 52.55471302234951
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 69.40657921763047
      },
      "scientific_discovery": {
       "accuracy": 0.8695652173913043,
       "cc": 86.73550472317316,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": 100,
        "score": 74.6992677927079
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 86.73550472317316
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.71909941626893,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.1572982488068
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.71909941626893
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.1809192085414,
      92.77187467791587
     ],
     "I": [
      76.33680965172114,
      83.43172085992364
     ],
     "capability": [
      82.6816915911483,
      87.75097702137747
     ],
     "composite": {
      "A": [
       73.29668924375432,
       75.30976141875034
      ],
      "B": [
       73.94546816229918,
       76.78676004590665
      ],
      "C": [
       73.29668924375432,
       75.30976141875034
      ]
     },
     "seed": 76
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.69553288708067,
     "capability": 85.85226550138285,
     "intelligence": 80.00899811568502
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8986666666666666,
      "calibration": 92.63072104192983,
      "cc": 86.91134541418701,
      "cc_by_type": {
       "choice": 86.91134541418701
      },
      "cc_tiered": 87.75435266858534,
      "cc_tiered_open": 89.43664280726739,
      "cc_tiered_sealed": 86.07206252990329,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 86.91134541418701
     },
     "noul": {
      "accuracy": 0.9253333333333333,
      "calibration": 91.9027642520753,
      "cc": 85.06666666666666,
      "cc_by_type": {
       "noul": 85.06666666666666
      },
      "cc_tiered": 82.94917938692693,
      "cc_tiered_open": 80.56521739130434,
      "cc_tiered_sealed": 85.33314138254951,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 85.06666666666666
     },
     "score": {
      "accuracy": 0.8266666666666667,
      "calibration": 90.5531133672369,
      "cc": 72.227609837006,
      "cc_by_type": {
       "score": 72.227609837006
      },
      "cc_tiered": 69.3234622915428,
      "cc_tiered_open": 66.73931227289297,
      "cc_tiered_sealed": 71.9076123101926,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 72.227609837006
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.002666666666666706,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.16486213201394,
    "Emphasis on Accuracy 60:20:20": 73.7967813820553,
    "Emphasis on Cost 20:20:60": 63.923143914690996,
    "Emphasis on Speed 20:60:20": 73.73581086017921,
    "Intelligence only": 80.00899811568502
   }
  },
  {
   "adapter_id": "images/smalljev/spec.json:remote_inproc_v15.RemoteInprocV15Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Aditya (isHeSatoshi)",
   "axes": {
    "calibration": 77.23092648324078,
    "cost": 60.77469204714053,
    "intelligence": 8.670533657732655,
    "speed": 90.11972690216095
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1267499372828752,
      "ece_score": 74.65001254342496,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 74.65001254342496,
      "score_before_time_limit": 74.65001254342496,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.7653333333333333,
      "brier": 0.252171281098217,
      "ece": 0.15089599377413593,
      "n": 375,
      "score": 69.82080124517282,
      "score_before_time_limit": 69.82080124517282,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9733333333333334,
      "interval90_width": 0.8366666666666663,
      "n": 375,
      "nrps": 0.15538624289679162,
      "score": 87.22196566112457,
      "score_before_time_limit": 87.22196566112457,
      "time_limit_failed_share": 0,
      "top_ece": 0.05008722194035847
     }
    },
    "score": 77.23092648324078
   },
   "capability": 42.95073007048671,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.24656367704805998,
     1.643207687945247
    ],
    "B": [
     0.1702968609786506,
     1.2179490503049588
    ],
    "C": [
     0.17122477572781938,
     1.1411164499619768
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020300665024630543
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "smalljev semantic-v9",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 10.044883698522286,
    "I_sealed": 7.296183616943025,
    "base": 8.670533657732655,
    "excess": 0.17531243914101724,
    "gap": 2.748700081579261,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 18.8258695634495,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 29.945945945945947,
       "hard": 13.180245360176167,
       "judge": 28.80685563612393,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -71.42857142857143,
       "hard": -73.91304347826086,
       "judge": -80,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 11.308781532117358,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 1.465713661728496,
       "hard": 15.552501797814056,
       "judge": 21.821381656800277,
       "standard": -8.026025251105985
      }
     },
     "sealed|choice": {
      "cc": 12.092078018745205,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 37.47690488507872,
       "hard": 8.37104072398188,
       "judge": 6.890130353817524,
       "standard": 14.644660672496622
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -60,
       "hard": -82.52427184466019,
       "judge": -52.63157894736843,
       "standard": -52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 9.796472832083872,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 12.348066695721426,
       "hard": 6.731683495180563,
       "judge": 16.491346453277444,
       "standard": 4.607944142281362
      }
     }
    }
   },
   "jevbench_score": 0.7718897362808083,
   "key": "smalljev",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7272727272727273,
    "decisive_rate": 0.23466666666666666,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.23092648324078,
     "cost": 60.77469204714053,
     "intelligence": 8.670533657732655,
     "speed": 90.21131723708834
    },
    "board_release": "v1.6.43",
    "capability": 42.95073007048671,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 107,
    "capability_rank": 135,
    "composite_ci95": {
     "A": [
      0.2465766163795243,
      1.6433530428400023
     ],
     "B": [
      0.17030179892922623,
      1.2180129321235638
     ],
     "C": [
      0.17123376137466964,
      1.1412173908611125
     ]
    },
    "jevbench_score": 0.7719455446134506,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 157,
    "rank": 200,
    "ranks": {
     "A": 200,
     "B": 200,
     "C": 200,
     "capability": 198
    },
    "scores": {
     "A": 0.7719455446134506,
     "B": 0.5545056442894855,
     "C": 0.5360732948704519
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2831438841298223,
     "p50_s_raw": 0.06657194206491113,
     "p95_s_adjusted": 0.33640395337715745,
     "p95_s_raw": 0.09320197668857871
    },
    "views": {
     "Balanced 33:33:33": 0.6314288092197278,
     "Emphasis on Accuracy 60:20:20": 0.4025186185800094,
     "Emphasis on Cost 20:20:60": 0.8553620766195851,
     "Emphasis on Speed 20:60:20": 0.9110150146047675,
     "Intelligence only": 0.2607338855207101
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 241,
   "ranked": true,
   "ranks": {
    "A": 241,
    "B": 241,
    "C": 241,
    "capability": 238
   },
   "repo": "https://github.com/isHeSatoshi/smalljev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.7718897362808083,
    "B": 0.5544826064895818,
    "C": 0.5360345390838948
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.28393019186332824,
    "p50_s_raw": 0.06696509593166411,
    "p95_s_adjusted": 0.34262238498777153,
    "p95_s_raw": 0.09631119249388576
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "MiniCPM5-2B-Base, 2.5B dense, with LoRA and native semantic decision heads",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.4661928849833,
      79.90983904292283
     ],
     "I": [
      5.751868300889253,
      11.443348446757808
     ],
     "capability": [
      39.42005219873977,
      44.84742177133798
     ],
     "composite": {
      "A": [
       0.24656367704805998,
       1.643207687945247
      ],
      "B": [
       0.1702968609786506,
       1.2179490503049588
      ],
      "C": [
       0.17122477572781938,
       1.1411164499619768
      ]
     },
     "seed": 101
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.23092648324078,
     "capability": 42.95073007048671,
     "intelligence": 8.670533657732655
    },
    "per_type": {
     "choice": {
      "accuracy": 0.336,
      "calibration": 74.65001254342496,
      "cc": 15.0838835576859,
      "cc_by_type": {
       "choice": 15.0838835576859
      },
      "cc_tiered": 15.458973791097353,
      "cc_tiered_open": 18.8258695634495,
      "cc_tiered_sealed": 12.092078018745205,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 15.0838835576859
     },
     "noul": {
      "accuracy": 0.17066666666666666,
      "calibration": 69.82080124517282,
      "cc": -65.86666666666667,
      "cc_by_type": {
       "noul": -65.86666666666667
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.3626666666666667,
      "calibration": 87.22196566112457,
      "cc": 9.603703249850426,
      "cc_by_type": {
       "score": 9.603703249850426
      },
      "cc_tiered": 10.552627182100615,
      "cc_tiered_open": 11.308781532117358,
      "cc_tiered_sealed": 9.796472832083872,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 9.603703249850426
     }
    },
    "raw_on_A": {
     "calibration": 71.91610164610462,
     "capability": 40.0721638349662,
     "intelligence": 8.22822602382777
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.6313790228575082,
    "Emphasis on Accuracy 60:20:20": 0.40250647893597635,
    "Emphasis on Cost 20:20:60": 0.8553072591161187,
    "Emphasis on Speed 20:60:20": 0.9108284924171282,
    "Intelligence only": 0.2607338855207101
   }
  },
  {
   "adapter_id": "images/spark-s1-4b-v6/spec.json:spark_v15.Spark",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Abhishek Rai (abhishek085)",
   "axes": {
    "calibration": 64.79562625454915,
    "cost": 74.96399197637558,
    "intelligence": 43.2364603010071,
    "speed": 86.33664750658791
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.20104071779836963,
      "ece_score": 59.79185644032607,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 59.79185644032607,
      "score_before_time_limit": 59.79185644032607,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.16000000000000003,
      "brier": 0.16425289707950683,
      "ece": 0.1444944938079116,
      "n": 375,
      "score": 71.10110123841768,
      "score_before_time_limit": 71.10110123841768,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.712,
      "interval90_width": 0.17133333333333342,
      "n": 375,
      "nrps": 0.1542624452699586,
      "score": 63.493921084903704,
      "score_before_time_limit": 63.493921084903704,
      "time_limit_failed_share": 0,
      "top_ece": 0.2879295665159836
     }
    },
    "score": 64.79562625454915
   },
   "capability": 54.016043277778124,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,129,092 input + 0 output tokens over 1,624 decisions = 695.253694581 tokens/decision. GPU-time cost USD 0.00683177572476398/1,000; old list-price cost USD 0.0208576108374384/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00683177572476398/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.006831775724763977
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "spark-s1-4b-v6 (Open Spark Jev, abhishek085)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 47.64157364296756,
    "I_sealed": 38.83134695904664,
    "base": 43.2364603010071,
    "excess": 6.236839041482673,
    "gap": 8.810226683920916,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 63.30038752190768,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 58.47750865051904,
       "judge": 68.35860250494396,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 46.0848861283644,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 56.52173913043479,
       "judge": 10.000000000000009,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 33.539447278630604,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 38.99774322763003,
       "hard": 44.04391027813551,
       "judge": 21.602744427042197,
       "standard": 27.706427582503657
      }
     },
     "sealed|choice": {
      "cc": 57.98194607005097,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 72.06414899120539,
       "hard": 50.113122171945705,
       "judge": 60.89385474860336,
       "standard": 62.31062938785566
      }
     },
     "sealed|noul": {
      "cc": 33.87772460002735,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 56.00000000000001,
       "hard": 2.9126213592232997,
       "judge": 50,
       "standard": 60.56338028169015
      }
     },
     "sealed|score": {
      "cc": 24.634370207061615,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 51.84151834556672,
       "hard": 16.688097933910274,
       "judge": 26.27544811817185,
       "standard": 24.46172381744639
      }
     }
    }
   },
   "jevbench_score": 47.114595108116106,
   "key": "spark-s1-4b-v6",
   "last_measured_on": "2026-10-05",
   "licence": "Apache-2.0 (code and weights); base Qwen/Qwen3.5-4B Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8317460317460318,
    "decisive_rate": 0.84,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 64.79562625454915,
     "cost": 60.422063196854964,
     "intelligence": 43.2364603010071,
     "speed": 86.95618624402076
    },
    "board_release": "v1.6.43",
    "capability": 54.016043277778124,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 86,
    "capability_rank": 112,
    "composite_ci95": {
     "A": [
      29.776291757874876,
      55.39058631138699
     ],
     "B": [
      26.921562908696533,
      52.0349558138311
     ],
     "C": [
      20.677980387413104,
      38.465684938463184
     ]
    },
    "jevbench_score": 44.90217636528639,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 46,
    "rank": 71,
    "ranks": {
     "A": 71,
     "B": 73,
     "C": 75,
     "capability": 167
    },
    "scores": {
     "A": 44.90217636528639,
     "B": 41.66207269525555,
     "C": 31.18206692033776
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.40721648099715824,
     "p50_s_raw": 0.1286082404985791,
     "p95_s_adjusted": 0.4949439602918574,
     "p95_s_raw": 0.17247198014592868
    },
    "views": {
     "Balanced 33:33:33": 43.83191134540322,
     "Emphasis on Accuracy 60:20:20": 38.371611426428935,
     "Emphasis on Cost 20:20:60": 44.36176060417368,
     "Emphasis on Speed 20:60:20": 50.402171180021895,
     "Intelligence only": 32.33034853965725
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 125,
   "ranked": true,
   "ranks": {
    "A": 125,
    "B": 127,
    "C": 130,
    "capability": 202
   },
   "repo": "https://github.com/abhishek085/open-spark-jev",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 47.114595108116106,
    "B": 43.16668292791795,
    "C": 32.71846882508062
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4085616710057366,
    "p50_s_raw": 0.12928083550286829,
    "p95_s_adjusted": 0.5689546199035249,
    "p95_s_raw": 0.20947730995176245
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B with a LoRA, read at the option-letter logits with a fitted temperature (no extra head)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      59.71050928645707,
      68.28479086272019
     ],
     "I": [
      36.48865188869242,
      47.17023357174763
     ],
     "capability": [
      48.85711252205833,
      57.24976166234473
     ],
     "composite": null,
     "seed": 102
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 64.79562625454915,
     "capability": 54.016043277778124,
     "intelligence": 43.2364603010071
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6866666666666666,
      "calibration": 59.79185644032607,
      "cc": 59.92914183947025,
      "cc_by_type": {
       "choice": 59.92914183947025
      },
      "cc_tiered": 60.64116679597932,
      "cc_tiered_open": 63.30038752190768,
      "cc_tiered_sealed": 57.98194607005097,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 59.92914183947025
     },
     "noul": {
      "accuracy": 0.6986666666666667,
      "calibration": 71.10110123841768,
      "cc": 39.733333333333334,
      "cc_by_type": {
       "noul": 39.733333333333334
      },
      "cc_tiered": 39.981305364195876,
      "cc_tiered_open": 46.0848861283644,
      "cc_tiered_sealed": 33.87772460002735,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.733333333333334
     },
     "score": {
      "accuracy": 0.5386666666666666,
      "calibration": 63.493921084903704,
      "cc": 27.536544538601227,
      "cc_by_type": {
       "score": 27.536544538601227
      },
      "cc_tiered": 29.08690874284611,
      "cc_tiered_open": 33.539447278630604,
      "cc_tiered_sealed": 24.634370207061615,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.536544538601227
     }
    },
    "raw_on_A": {
     "calibration": 68.7404409420719,
     "capability": 57.96450731310942,
     "intelligence": 47.18857368414694
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 46.68524992270459,
    "Emphasis on Accuracy 60:20:20": 39.64430941081177,
    "Emphasis on Cost 20:20:60": 50.030276608987386,
    "Emphasis on Speed 20:60:20": 52.49915054907653,
    "Intelligence only": 32.33034853965725
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": true,
   "api_subset": false,
   "author": "SPX AI team (SurdAI, formerly Simplex AI)",
   "axes": {
    "calibration": 90.20036269195566,
    "cost": 52.107175358413464,
    "intelligence": 58.43051843707937,
    "speed": 79.18845201110972
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05772702157730632,
      "ece_score": 88.45459568453875,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.45459568453875,
      "score_before_time_limit": 88.45459568453875,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.18598382749326148,
      "brier": 0.08492257448407475,
      "ece": 0.04230529224114798,
      "n": 371,
      "score": 91.5389415517704,
      "score_before_time_limit": 91.5389415517704,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9322493224932249,
      "interval90_width": 0.30013550135501355,
      "n": 369,
      "nrps": 0.06769358396529002,
      "score": 90.60755083955786,
      "score_before_time_limit": 90.60755083955786,
      "time_limit_failed_share": 0,
      "top_ece": 0.06007769962177635
     }
    },
    "score": 90.20036269195566
   },
   "capability": 74.31544056451752,
   "class": "Hosted decision API",
   "composite_ci95": {
    "A": [
     64.32849489209721,
     67.89404426087012
    ],
    "B": [
     61.78442076552503,
     66.80330225899888
    ],
    "C": [
     50.560624091643184,
     67.89404426087012
    ]
   },
   "cost": {
    "basis": "Operator's own public-beta overage tariff, conservative reading USD 0.04 per 1M input tokens, output USD 0 (the provider's /v1/models payload reports output_price \"0\"): three dated primary readings of the operator's own published overage tariff: 24 Sep 2026 provider console (flash 0.025, pro 0.08 per 1M input), 1 Oct 2026 docs page (flash 0.04, pro 0.02), 6 Oct 2026 docs page AND /v1/models payload agreeing (flash 0.02, pro 0.04). Sources conflict, so the per-model MAXIMUM is used, which is the published fastino-gliner-2-5-decide precedent (\"published sources conflict ... conservative higher\"). Today's agreeing pair would be USD 0.02/M, i.e. half this row's cost. No base-model floor applies: the submitter states the base model stays sealed until a later open release, so no base model exists to reference. Priced on usage.input_tokens as the provider reports them. The provider also reports billable_input_tokens = input_tokens / 2 on every single item (297/297 of the public set, exactly 2.0): its default effort 2 renders the prompt twice and its docs bundle the second pass free during public beta (\"default effort 2 is included at the single-pass input price\"). That bundle is a public-beta promotion, and TASK.md forbids a free or promotional tier setting the price, so the two passes actually consumed are charged. The operator's own billed figure is exactly half of this row's cost",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.03887914666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.039484576844955994
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SPX-CD Flash (SurdAI, hosted /v1/systemone, Oct-4 checkpoint)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0 S u P (1,500 rows) under the published 5 Oct 2026 full-set amendment, through the standard full-set API lane (run_api_full_v161.py, rotation.py expose --full-set logged 1,200 sealed items to provider 'surd' before the first request); stock jevbench.adapters.typesafe with an endpoint override only, reasoning at the provider default, one request at a time, no retries; add-requests run 53, benchmarkheaven.com/submit ref 3FF00545",
   "endpoint_kind": "api",
   "full_coverage": true,
   "gpu": null,
   "intelligence": {
    "I_open": 59.99669320470862,
    "I_sealed": 56.86434366945013,
    "base": 58.43051843707937,
    "excess": 0.5589618928202476,
    "gap": 3.1323495352584914,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.41749935055667,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396666,
       "judge": 68.35860250494396,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 52.89855072463768,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 0,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 51.67402953893152,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.86207439440385,
       "hard": 52.93207079861204,
       "judge": 54.720404668190085,
       "standard": 42.994361897946476
      }
     },
     "sealed|choice": {
      "cc": 71.02357528674706,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 67.92986425339366,
       "judge": 68.34264432029796,
       "standard": 76.7212710924991
      }
     },
     "sealed|noul": {
      "cc": 45.34763039144423,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 20.388349514563096,
       "judge": 52.63157894736843,
       "standard": 69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 54.221825330159106,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 80.5246722641586,
       "hard": 48.238601087091794,
       "judge": 55.5539283927005,
       "standard": 51.038695755481854
      }
     }
    }
   },
   "jevbench_score": 66.64434875110587,
   "key": "spx-cd-flash",
   "last_measured_on": "2026-10-06",
   "licence": null,
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "provider-reported measurement model slug: spx-cd-flash",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9370860927152318,
    "decisive_rate": 0.8140161725067385,
    "supported": true,
    "valid": 371
   },
   "open": "closed weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 21,
    "axes": {
     "calibration": 90.20036269195566,
     "cost": 52.107175358413464,
     "intelligence": 58.43051843707937,
     "speed": 79.39117060033084
    },
    "board_release": "v1.6.43",
    "capability": 74.31544056451752,
    "capability_api_board_rank": 12,
    "capability_open_board_rank": null,
    "capability_rank": 26,
    "composite_ci95": {
     "A": [
      64.36187068361473,
      67.93122351228547
     ],
     "B": [
      61.80904827802077,
      66.83209430434223
     ],
     "C": [
      50.5869412571725,
      67.93122351228547
     ]
    },
    "jevbench_score": 66.68017155386212,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 44,
    "ranks": {
     "A": 44,
     "B": 43,
     "C": 38,
     "capability": 48
    },
    "scores": {
     "A": 66.68017155386212,
     "B": 64.84899920515487,
     "C": 63.23735362072274
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.9804960675537586,
     "p50_s_raw": 0.9804960675537586,
     "p95_s_adjusted": 1.1733756832778452,
     "p95_s_raw": 1.1733756832778452
    },
    "views": {
     "Balanced 33:33:33": 61.34791411142523,
     "Emphasis on Accuracy 60:20:20": 60.146681111986254,
     "Emphasis on Cost 20:20:60": 57.28436724874348,
     "Emphasis on Speed 20:60:20": 67.48263394824296,
     "Intelligence only": 58.43051843707938
    }
   },
   "provenance": {
    "carried_metadata_from": "Current-pool API measurement registry",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 102,
   "ranked": true,
   "ranks": {
    "A": 102,
    "B": 94,
    "C": 77,
    "capability": 66
   },
   "repo": "https://surd.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 66.64434875110587,
    "B": 64.82189014379784,
    "C": 63.20338041410385
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.0145484656095505,
    "p50_s_raw": 1.0145484656095505,
    "p95_s_adjusted": 1.18817935064435,
    "p95_s_raw": 1.18817935064435
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.82360765229315,
      91.10972764372464
     ],
     "I": [
      53.013471055498634,
      62.79232050259918
     ],
     "capability": [
      69.82626986507542,
      76.43255727254187
     ],
     "composite": {
      "A": [
       64.32849489209721,
       67.89404426087012
      ],
      "B": [
       61.78442076552503,
       66.80330225899888
      ],
      "C": [
       50.560624091643184,
       67.89404426087012
      ]
     },
     "seed": 136
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.20036269195566,
     "capability": 74.31544056451752,
     "intelligence": 58.43051843707937
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7893333333333333,
      "calibration": 88.45459568453875,
      "cc": 73.05874217292042,
      "cc_by_type": {
       "choice": 73.05874217292042
      },
      "cc_tiered": 73.22053731865186,
      "cc_tiered_open": 75.41749935055667,
      "cc_tiered_sealed": 71.02357528674706,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 73.05874217292042
     },
     "noul": {
      "accuracy": 0.7546666666666667,
      "calibration": 91.5389415517704,
      "cc": 50.933333333333344,
      "cc_by_type": {
       "noul": 50.933333333333344
      },
      "cc_tiered": 49.12309055804096,
      "cc_tiered_open": 52.89855072463768,
      "cc_tiered_sealed": 45.34763039144423,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 50.933333333333344
     },
     "score": {
      "accuracy": 0.6746666666666666,
      "calibration": 90.60755083955786,
      "cc": 54.63501832116133,
      "cc_by_type": {
       "score": 54.63501832116133
      },
      "cc_tiered": 52.94792743454531,
      "cc_tiered_open": 51.67402953893152,
      "cc_tiered_sealed": 54.221825330159106,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.63501832116133
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 61.307488965987744,
    "Emphasis on Accuracy 60:20:20": 60.12336025933381,
    "Emphasis on Cost 20:20:60": 57.26321281799819,
    "Emphasis on Speed 20:60:20": 67.39464498875742,
    "Intelligence only": 58.43051843707938
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "SPX AI team (SurdAI)",
   "axes": {
    "calibration": 87.47801599949784,
    "cost": 68.67068018166556,
    "intelligence": 48.779308810848946,
    "speed": 89.06649387039909
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04837927211703187,
      "ece_score": 90.32414557659362,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 90.32414557659362,
      "score_before_time_limit": 90.32414557659362,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3207547169811321,
      "brier": 0.10397941406905582,
      "ece": 0.08441105054816544,
      "n": 371,
      "score": 83.11778989036691,
      "score_before_time_limit": 83.11778989036691,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.975609756097561,
      "interval90_width": 0.31187895212285444,
      "n": 369,
      "nrps": 0.06867686607554577,
      "score": 88.99211253153302,
      "score_before_time_limit": 88.99211253153302,
      "time_limit_failed_share": 0,
      "top_ece": 0.07574044164689696
     }
    },
    "score": 87.47801599949784
   },
   "capability": 68.1286624051734,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 726,752 input + 0 output tokens over 1,477 decisions = 492.046039269 tokens/decision. GPU-time cost USD 0.0110741566005038/1,000; old list-price cost USD 0.0246023019634394/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0110741566005038/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.024225066666666666
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011074156600503763
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SPX-CD-Omni (SurdAI, google/gemma-4-12B-it + LoRA r32, one forward pass, self-hosted)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0 S u P (1,500 rows, 1,477 ok; the 23 refusals are exactly the 23 long-context items, through the author's own \"Prompt exceeds context limit; no automatic truncation\" guard at their shipped --max-context 16384, although the base model itself declares 262,144 positions). Author's own code unmodified (infer.py / decision.py, sha256 1c5db5de.../43389f44...) with their shipped --backend transformers on the unquantized base and the flags their own EVALUATION.md prescribes FOR JEVBENCH: --prompt-format open-format --effort 1 --temperature 1.0. Their requirements-transformers.txt is missing torchvision, which transformers 5.16.1 needs for the Gemma 4 unified processor; adding it was the only deviation and it changes no numerics. One RTX PRO 6000 Blackwell (Lium, Brasov, Romania), torch 2.12.0+cu130, transformers 5.16.1, peft 0.20.0. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment; raw p50/p95 are in the run 53 receipts; add-requests run 53, benchmarkheaven.com/submit ref 75D5EF0C",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000 Blackwell",
   "intelligence": {
    "I_open": 47.456697185695944,
    "I_sealed": 50.10192043600195,
    "base": 48.779308810848946,
    "excess": -5.2186108927442465,
    "gap": -2.645223250306003,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 69.72620969563555,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 69.80182447310474,
       "judge": 68.35860250494396,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 16.937888198757765,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 39.13043478260869,
       "judge": -9.999999999999998,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 55.70599366269451,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 45.44648364886738,
       "hard": 61.913039440117416,
       "judge": 59.403373958799065,
       "standard": 42.875586670605394
      }
     },
     "sealed|choice": {
      "cc": 73.93119843479649,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 84.03665656640308,
       "hard": 69.45701357466062,
       "judge": 73.9292364990689,
       "standard": 77.82978199285628
      }
     },
     "sealed|noul": {
      "cc": 24.24849331039893,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 32.00000000000001,
       "hard": 4.854368932038833,
       "judge": 28.947368421052634,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 52.12606956281044,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 64.31573995848055,
       "hard": 49.879926355330106,
       "judge": 59.15248322255222,
       "standard": 39.98390029032339
      }
     }
    }
   },
   "jevbench_score": 65.95560252970363,
   "key": "spx-cd-omni",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "SurdAI/SPX-CD-Omni@06a786a13f2ce9b8d1b8f541cf030fae007008fb + google/gemma-4-12B-it@707f0a3b8a3c7ad586ed01e27eafbad8a27dd0f7",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9285714285714286,
    "decisive_rate": 0.6792452830188679,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.47801599949784,
     "cost": 58.27072766098771,
     "intelligence": 48.779308810848946,
     "speed": 89.44891259349035
    },
    "board_release": "v1.6.43",
    "capability": 68.1286624051734,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 32,
    "capability_rank": 53,
    "composite_ci95": {
     "A": [
      49.00994204245296,
      68.5754476769007
     ],
     "B": [
      44.94378680804397,
      65.03574713274071
     ],
     "C": [
      34.03468197392567,
      55.062399131442234
     ]
    },
    "jevbench_score": 63.16406085537614,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 30,
    "rank": 51,
    "ranks": {
     "A": 51,
     "B": 55,
     "C": 59,
     "capability": 90
    },
    "scores": {
     "A": 63.16406085537614,
     "B": 58.9160449775072,
     "C": 43.86393114956675
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2966954869683832,
     "p50_s_raw": 0.0733477434841916,
     "p95_s_adjusted": 0.38264654851518576,
     "p95_s_raw": 0.11632327425759287
    },
    "views": {
     "Balanced 33:33:33": 58.4608318148256,
     "Emphasis on Accuracy 60:20:20": 52.9688184573952,
     "Emphasis on Cost 20:20:60": 57.22245674010682,
     "Emphasis on Speed 20:60:20": 66.8372282634097,
     "Intelligence only": 46.42660407688444
    }
   },
   "provenance": {
    "carried_metadata_from": "r53 PREREG-R53.md, dl_r53.sh and current registry",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 105,
   "ranked": true,
   "ranks": {
    "A": 105,
    "B": 106,
    "C": 111,
    "capability": 116
   },
   "repo": "https://huggingface.co/SurdAI/SPX-CD-Omni",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 65.95560252970363,
    "B": 60.83744101407832,
    "C": 45.80250175673862
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3001056468114257,
    "p50_s_raw": 0.07505282340571284,
    "p95_s_adjusted": 0.4131202141754329,
    "p95_s_raw": 0.13156010708771645
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.85250104999506,
      89.14284725267011
     ],
     "I": [
      43.845760787268546,
      53.75726553555249
     ],
     "capability": [
      64.04521719808471,
      70.65155471508619
     ],
     "composite": null,
     "seed": 137
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.47801599949784,
     "capability": 68.1286624051734,
     "intelligence": 48.779308810848946
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7973333333333333,
      "calibration": 90.32414557659362,
      "cc": 74.08182791318927,
      "cc_by_type": {
       "choice": 74.08182791318927
      },
      "cc_tiered": 71.82870406521602,
      "cc_tiered_open": 69.72620969563555,
      "cc_tiered_sealed": 73.93119843479649,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 74.08182791318927
     },
     "noul": {
      "accuracy": 0.624,
      "calibration": 83.11778989036691,
      "cc": 24.8,
      "cc_by_type": {
       "noul": 24.8
      },
      "cc_tiered": 20.593190754578348,
      "cc_tiered_open": 16.937888198757765,
      "cc_tiered_sealed": 24.24849331039893,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.8
     },
     "score": {
      "accuracy": 0.6613333333333333,
      "calibration": 88.99211253153302,
      "cc": 52.652284619387025,
      "cc_by_type": {
       "score": 52.652284619387025
      },
      "cc_tiered": 53.91603161275248,
      "cc_tiered_open": 55.70599366269451,
      "cc_tiered_sealed": 52.12606956281044,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.652284619387025
     }
    },
    "raw_on_A": {
     "calibration": 83.85218691107563,
     "capability": 66.88160846988468,
     "intelligence": 49.91103002869373
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 61.68255687465877,
    "Emphasis on Accuracy 60:20:20": 54.51678984059403,
    "Emphasis on Cost 20:20:60": 63.10220406932833,
    "Emphasis on Speed 20:60:20": 69.22409923434977,
    "Intelligence only": 46.42660407688444
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": true,
   "api_subset": false,
   "author": "SPX AI team (SurdAI, formerly Simplex AI)",
   "axes": {
    "calibration": 92.33141939321372,
    "cost": 43.076275488494026,
    "intelligence": 64.6277726176258,
    "speed": 77.43431239288299
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.037067490257276604,
      "ece_score": 92.58650194854468,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 92.58650194854468,
      "score_before_time_limit": 92.58650194854468,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.1617250673854448,
      "brier": 0.07013566233777668,
      "ece": 0.037583902439496455,
      "n": 371,
      "score": 92.48321951210072,
      "score_before_time_limit": 92.48321951210072,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.983739837398374,
      "interval90_width": 0.26445347786811196,
      "n": 369,
      "nrps": 0.05200753435023827,
      "score": 91.92453671899578,
      "score_before_time_limit": 92.17032425033,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.052292990321580954
     }
    },
    "score": 92.33141939321372
   },
   "capability": 78.47959600541975,
   "class": "Hosted decision API",
   "composite_ci95": {
    "A": [
     46.27333224964364,
     48.08682307157107
    ],
    "B": [
     45.84268883284009,
     48.63626352048142
    ],
    "C": [
     45.367953339258825,
     48.08682307157107
    ]
   },
   "cost": {
    "basis": "Operator's own public-beta overage tariff, conservative reading USD 0.08 per 1M input tokens, output USD 0: three dated primary readings of the operator's own published overage tariff: 24 Sep 2026 provider console (flash 0.025, pro 0.08 per 1M input), 1 Oct 2026 docs page (flash 0.04, pro 0.02), 6 Oct 2026 docs page AND /v1/models payload agreeing (flash 0.02, pro 0.04). Sources conflict, so the per-model MAXIMUM is used, which is the published fastino-gliner-2-5-decide precedent (\"published sources conflict ... conservative higher\"). Today's agreeing pair would be USD 0.04/M, i.e. half this row's cost. No base-model floor applies: the submitter states the base model stays sealed until a later open release. Priced on usage.input_tokens as the provider reports them. The provider also reports billable_input_tokens = input_tokens / 2 on every single item (297/297 of the public set, exactly 2.0): its default effort 2 renders the prompt twice and its docs bundle the second pass free during public beta (\"default effort 2 is included at the single-pass input price\"). That bundle is a public-beta promotion, and TASK.md forbids a free or promotional tier setting the price, so the two passes actually consumed are charged. The operator's own billed figure is exactly half of this row's cost",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.07775829333333333
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.07896915368991199
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SPX-CD Pro (SurdAI, hosted /v1/systemone, Oct-4 checkpoint)",
   "endpoint_condition": "measured 2026-10-06 on the live v1.6.0 S u P (1,500 rows) under the published 5 Oct 2026 full-set amendment, same lane, same adapter and same provider default as spx-cd-flash; add-requests run 53, benchmarkheaven.com/submit ref 3FF00545",
   "endpoint_kind": "api",
   "full_coverage": true,
   "gpu": null,
   "intelligence": {
    "I_open": 65.26225762662179,
    "I_sealed": 63.993287608629814,
    "base": 64.6277726176258,
    "excess": -1.3044176244462684,
    "gap": 1.2689700179919754,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 77.44982804388046,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 79.23875432525952,
       "judge": 80.22412656558997,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 58.726017943409246,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 19.999999999999996,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 59.61092689257566,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.50227372429029,
       "hard": 68.13445392047002,
       "judge": 54.627278742003114,
       "standard": 48.59367164678844
      }
     },
     "sealed|choice": {
      "cc": 78.81397986232295,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 74.5475113122172,
       "judge": 77.6536312849162,
       "standard": 87.80638009607095
      }
     },
     "sealed|noul": {
      "cc": 53.70130193527029,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 37.86407766990292,
       "judge": 52.63157894736843,
       "standard": 71.83098591549295
      }
     },
     "sealed|score": {
      "cc": 59.46458102829621,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.86467136136848,
       "hard": 54.98115972463263,
       "judge": 60.73406209967376,
       "standard": 62.82715686202089
      }
     }
    }
   },
   "jevbench_score": 47.553774484141584,
   "key": "spx-cd-pro",
   "last_measured_on": "2026-10-06",
   "licence": null,
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "provider-reported measurement model slug: spx-cd-pro",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9517684887459807,
    "decisive_rate": 0.8382749326145552,
    "supported": true,
    "valid": 371
   },
   "open": "closed weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 25,
    "axes": {
     "calibration": 92.41334857032513,
     "cost": 43.076275488494026,
     "intelligence": 64.85693822910903,
     "speed": 77.96292698098645
    },
    "board_release": "v1.6.43",
    "capability": 78.63514339971708,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      46.40578610997944,
      48.21089390268207
     ],
     "B": [
      45.98130956469153,
      48.76321124599849
     ],
     "C": [
      45.82343084108345,
      48.21089390268207
     ]
    },
    "jevbench_score": 47.66970820706367,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 68,
    "ranks": {
     "A": 68,
     "B": 65,
     "C": 55,
     "capability": 32
    },
    "scores": {
     "A": 47.66970820706367,
     "B": 47.762738448829694,
     "C": 47.66970820706367
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 1.1281277425587177,
     "p50_s_raw": 1.1281277425587177,
     "p95_s_adjusted": 1.4169320456683636,
     "p95_s_raw": 1.4169320456683636
    },
    "views": {
     "Balanced 33:33:33": 43.27025827744028,
     "Emphasis on Accuracy 60:20:20": 45.09442248470386,
     "Emphasis on Cost 20:20:60": 37.91159972444074,
     "Emphasis on Speed 20:60:20": 48.12587216483719,
     "Intelligence only": 48.138519063832746
    }
   },
   "provenance": {
    "carried_metadata_from": "Current-pool API measurement registry",
    "kind": "v1.6.1 addendum a12 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 123,
   "ranked": true,
   "ranks": {
    "A": 123,
    "B": 120,
    "C": 107,
    "capability": 44
   },
   "repo": "https://surd.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 47.553774484141584,
    "B": 47.6361303647083,
    "C": 47.553774484141584
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.1453867703676224,
    "p50_s_raw": 1.1453867703676224,
    "p95_s_adjusted": 1.576219144836068,
    "p95_s_raw": 1.576219144836068
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.37372529307686,
      92.22160309760052
     ],
     "I": [
      59.366080391588525,
      68.65344947022643
     ],
     "capability": [
      74.24700490531005,
      79.69851586001273
     ],
     "composite": {
      "A": [
       46.27333224964364,
       48.08682307157107
      ],
      "B": [
       45.84268883284009,
       48.63626352048142
      ],
      "C": [
       45.367953339258825,
       48.08682307157107
      ]
     },
     "seed": 138
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 92.33141939321372,
     "capability": 78.47959600541975,
     "intelligence": 64.6277726176258
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8373333333333334,
      "calibration": 92.58650194854468,
      "cc": 79.1972566145335,
      "cc_by_type": {
       "choice": 79.1972566145335
      },
      "cc_tiered": 78.1319039531017,
      "cc_tiered_open": 77.44982804388046,
      "cc_tiered_sealed": 78.81397986232295,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 79.1972566145335
     },
     "noul": {
      "accuracy": 0.7893333333333333,
      "calibration": 92.48321951210072,
      "cc": 57.86666666666667,
      "cc_by_type": {
       "noul": 57.86666666666667
      },
      "cc_tiered": 56.21365993933976,
      "cc_tiered_open": 58.726017943409246,
      "cc_tiered_sealed": 53.70130193527029,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 57.86666666666667
     },
     "score": {
      "accuracy": 0.6826666666666666,
      "calibration": 91.92453671899578,
      "cc": 59.94377608650918,
      "cc_by_type": {
       "score": 59.94377608650918
      },
      "cc_tiered": 59.537753960435936,
      "cc_tiered_open": 59.61092689257566,
      "cc_tiered_sealed": 59.46458102829621,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 59.94377608650918
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 43.15098842304451,
    "Emphasis on Accuracy 60:20:20": 44.95698890795167,
    "Emphasis on Cost 20:20:60": 37.85659305616461,
    "Emphasis on Speed 20:60:20": 47.92862089900984,
    "Intelligence only": 47.968426341937814
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Standard Thinking",
   "axes": {
    "calibration": 78.35523487828182,
    "cost": 48.64218899335634,
    "intelligence": 32.78619016656576,
    "speed": 86.38605449679788
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12573842746652913,
      "ece_score": 74.85231450669417,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 74.85231450669417,
      "score_before_time_limit": 74.85231450669417,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3726541554959786,
      "brier": 0.17156906643303033,
      "ece": 0.09634973535153724,
      "n": 373,
      "score": 80.73005292969255,
      "score_before_time_limit": 80.73005292969255,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9086021505376344,
      "interval90_width": 0.44130824372759864,
      "n": 372,
      "nrps": 0.12094679093502676,
      "score": 79.48333719845877,
      "score_before_time_limit": 79.48333719845877,
      "time_limit_failed_share": 0,
      "top_ece": 0.14469323254789898
     }
    },
    "score": 78.35523487828182
   },
   "capability": 55.57071252242379,
   "composite_ci95": {
    "A": [
     12.825119517917724,
     27.43337606642157
    ],
    "B": [
     11.00758497617656,
     24.829735634421777
    ],
    "C": [
     8.906332998553973,
     19.05095560168164
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference (read 2026-10-12T00:14Z, source/public-docs/OPENROUTER-MINISTRAL-3-PRICES.json): the model card names base_model StandardThinking/StandardOne-3B, built on mistralai/Ministral-3-3B-Instruct-2512. Neither Standard One checkpoint is listed on OpenRouter; the exact underlying base is: OpenRouter mistralai/ministral-3b-2512 list price USD 0.10/M input and USD 0.10/M output. Self-hosted weights, no bookable tariff of the measured deployment; the author's stated hosted list price (USD 0.038/M input, standardthinking.ai/pricing) is below this floor, so the M2 §2 floor binds (price floor: base-model reference price applied). The schema head generates no text, so output tokens are 0 and priced at the reference output price. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.051514 per 1,000 decisions (0.797x the official cost cap USD 0.064595), against USD 0.050999 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05099906666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05151420875420875
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Standard One 3B SH (Standard Thinking)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container (--network none); author's reference server code/serve_head.py, Quick-start argv, unchanged defaults paid evaluation (public). Same customer and reviewed route as standard_one_8b_sh: official pytorch/pytorch:2.11.0-cuda12.8-cudnn9-runtime plus the documented requirements (transformers 5.12.1, peft 0.21.0) installed offline from SHA-256-pinned PyPI wheels (PR #299, bound to this order by PR #332). Requests over the documented 32,768-token context answer HTTP 422 (refused, not truncated). Customer list price USD 0.038/M input is below the M2 base floor.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 40.348148949812646,
    "I_sealed": 25.224231383318866,
    "base": 32.78619016656576,
    "excess": 5.284390796138069,
    "gap": 15.12391756649378,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 59.623482293580714,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 52.81535073922617,
       "judge": 48.58272907053395,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 26.282953761214635,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 19.999999999999996,
       "standard": 22.222222222222232
      }
     },
     "open|score": {
      "cc": 35.138010794642604,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 47.381015208512025,
       "hard": 33.253892427920064,
       "judge": 39.06078163000721,
       "standard": 26.90058906810605
      }
     },
     "sealed|choice": {
      "cc": 53.79293995999401,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 79.03430749682339,
       "hard": 40.152654099687744,
       "judge": 56.80705190989228,
       "standard": 63.931659987344446
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 17.24137931034482,
       "hard": -7.272727272727275,
       "judge": -23.076923076923073,
       "standard": 13.432835820895516
      }
     },
     "sealed|score": {
      "cc": 21.879754189962604,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 42.21631474162029,
       "hard": 12.573069905255352,
       "judge": 15.902582759068984,
       "standard": 39.2905996298887
      }
     }
    }
   },
   "jevbench_score": 21.588999608530766,
   "key": "standard_one_3b_sh",
   "last_measured_on": "2026-10-12",
   "licence": "Apache-2.0 (merged weights, schema head and reference server; base Ministral 3 3B Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.10",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "StandardThinking/StandardOne-3B-SH@bead20e94b7c96045a19dc4d43af48a66c8911ea",
   "noul_decisive": {
    "acc_among_decisive": 0.8418803418803419,
    "decisive": 234,
    "decisive_rate": 0.6273458445040214,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.35523487828182,
     "cost": 48.64218899335634,
     "intelligence": 32.78619016656576,
     "speed": 86.56241878672063
    },
    "board_release": "v1.6.43",
    "capability": 55.57071252242379,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 80,
    "capability_rank": 106,
    "composite_ci95": {
     "A": [
      12.828775574871745,
      27.44229986499225
     ],
     "B": [
      11.009739151550862,
      24.835586726192307
     ],
     "C": [
      8.908871926994268,
      19.057152684022387
     ]
    },
    "jevbench_score": 21.595754998730687,
    "measured_in": "v1.6.10",
    "method": "v1.6",
    "open_board_rank": 86,
    "rank": 119,
    "ranks": {
     "A": 119,
     "B": 133,
     "C": 136,
     "capability": 161
    },
    "scores": {
     "A": 21.595754998730687,
     "B": 19.217967771236903,
     "C": 14.997052082451866
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2202127271797508,
     "p50_s_raw": 0.0351063635898754,
     "p95_s_adjusted": 1.0021107152919284,
     "p95_s_raw": 0.42605535764596425
    },
    "views": {
     "Balanced 33:33:33": 19.4982986853078,
     "Emphasis on Accuracy 60:20:20": 16.460212427864356,
     "Emphasis on Cost 20:20:60": 19.615649720852925,
     "Emphasis on Speed 20:60:20": 23.737566720658748,
     "Intelligence only": 13.341942508299013
    }
   },
   "rank": 181,
   "ranked": true,
   "ranks": {
    "A": 181,
    "B": 183,
    "C": 184,
    "capability": 195
   },
   "repo": "https://huggingface.co/StandardThinking/StandardOne-3B-SH",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 21.588999608530766,
    "B": 19.213687633916003,
    "C": 14.992360839257477
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.22726393709890544,
    "p50_s_raw": 0.03863196854945272,
    "p95_s_adjusted": 1.0112627825932576,
    "p95_s_raw": 0.43063139129662886
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": " SUPERSEDED BY: standard_one_3b_sh_v4 (/jev-models/standard_one_3b_sh_v4); the predecessor measurement remains on this board.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5454545454545454,
       "cc": 32.77382596686048,
       "cc_by_type": {
        "choice": 25,
        "noul": 100,
        "score": -26.678522099418565
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 32.77382596686048
      },
      "calibration_gen": {
       "accuracy": 0.594059405940594,
       "cc": 38.90519048770406,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 36.00000000000001,
        "score": 28.435613880927658
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 38.90519048770406
      },
      "composite_gen": {
       "accuracy": 0.6039603960396039,
       "cc": 26.41990183209226,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": 3.703703703703698,
        "score": 1.5722619551747141
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 26.41990183209226
      },
      "fairness": {
       "accuracy": 0.6875,
       "cc": 54.227749261100236,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 42.68324778330073
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 54.227749261100236
      },
      "jaggedness": {
       "accuracy": 0.4111111111111111,
       "cc": 4.694910856583905,
       "cc_by_type": {
        "choice": 29.258902791145346,
        "noul": -40.54054054054054,
        "score": 25.366370319146913
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 4.694910856583905
      },
      "llm_router": {
       "accuracy": 0.7092651757188498,
       "cc": 56.50627669788855,
       "cc_by_type": {
        "choice": 59.584397288016206,
        "noul": 60.82474226804124,
        "score": 49.109690537608195
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 56.50627669788855
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -11.968674887015771,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75,
        "score": 13.320779462664035
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6,
       "cc": 27.719617249690184,
       "cc_by_type": {
        "choice": 61.85197261167266,
        "noul": -13.888888888888884,
        "score": 35.195768026286785
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 27.719617249690184
      },
      "negation_pair_gen": {
       "accuracy": 0.5697674418604651,
       "cc": 8.779651370862412,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": -46.666666666666664,
        "score": -10.327712554079426
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 8.779651370862412
      },
      "option_order": {
       "accuracy": 0.5833333333333334,
       "cc": 46.737683089214386,
       "cc_by_type": {
        "choice": 46.737683089214386
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 46.737683089214386
      },
      "perturb_pair": {
       "accuracy": 0.796875,
       "cc": 65.06712108997921,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 80,
        "score": 39.22538729396163
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 65.06712108997921
      },
      "usecase": {
       "accuracy": 0.4909090909090909,
       "cc": 11.828512149644082,
       "cc_by_type": {
        "choice": 53.43137254901962,
        "noul": -37.5,
        "score": 19.554163899912623
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 11.828512149644082
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -8.511711589251624,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": 0,
        "score": -11.249420482040584
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 50.290968909112884,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 17.539573394005338
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 50.290968909112884
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 77.36447447418108,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 55.987228732277764
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 77.36447447418108
      },
      "el": {
       "accuracy": 0.4,
       "cc": 20.54630558015352,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 95.34468955237863
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 20.54630558015352
      },
      "en": {
       "accuracy": 0.5716768027801912,
       "cc": 28.048815986073738,
       "cc_by_type": {
        "choice": 53.42990688422155,
        "noul": 6.474820143884896,
        "score": 24.241720930114763
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 28.048815986073738
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 71.78253394019436,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 25.135530042442756
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.78253394019436
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -16.069232069256785,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100,
        "score": 82.93984477583622
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5,
       "cc": 26.677036311333605,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 22.3840501104714
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 26.677036311333605
      },
      "hi": {
       "accuracy": 0.5714285714285714,
       "cc": 19.981627998541693,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25,
        "score": 18.46443706825078
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 19.981627998541693
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 68.19136996703737,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 18.603727983730966
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 68.19136996703737
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 22.28425616935985,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": -19.999999999999996,
        "score": 29.743951987031092
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 22.28425616935985
      },
      "ko": {
       "accuracy": 0.4,
       "cc": -12.485068762902236,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 6.989238155737743
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 74.38486199169195,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 89.82125264174256
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 74.38486199169195
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 21.807367811391327,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 24.729462741533283
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 21.807367811391327
      },
      "no": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5897435897435898,
       "cc": 20.034807741551614,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": -40,
        "score": 22.956729614836824
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 20.034807741551614
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 70.07017257534663,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 42.03137806781948
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 70.07017257534663
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 10.403452805008072,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 67.80610309587527
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 10.403452805008072
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 26.71730444108229,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 23.630174192812092
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 26.71730444108229
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 54.56527709507105,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 28.21196031747124
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 54.56527709507105
      },
      "zh": {
       "accuracy": 0.6,
       "cc": -10.14800160355746,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": -71.68694266377973
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.656,
       "cc": 55.567462063950714,
       "cc_by_type": {
        "choice": 55.567462063950714
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 55.567462063950714
      },
      "noul": {
       "accuracy": 0.5253333333333333,
       "cc": 5.066666666666664,
       "cc_by_type": {
        "noul": 5.066666666666664
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 5.066666666666664
      },
      "score": {
       "accuracy": 0.488,
       "cc": 26.316988637484396,
       "cc_by_type": {
        "score": 26.316988637484396
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 26.316988637484396
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": -6.4142401533742515,
       "cc_by_type": {
        "choice": 68,
        "noul": -75,
        "score": -12.242720460122758
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": 8.17797914598323,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -33.333333333333336,
        "score": 13.767891889295447
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 8.17797914598323
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 55.44150097547703,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 48.844828129683094
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 55.44150097547703
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -32.50493394907758,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -27.33936325074151
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5294117647058824,
       "cc": 15.833160915469014,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 9.818323326117174
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 15.833160915469014
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -48.17502336229969,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100,
        "score": -56.47475562149027
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5333333333333333,
       "cc": 6.404557415703015,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -71.42857142857143,
        "score": 20.054008381562827
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 6.404557415703015
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": 1.2325164356394527,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50,
        "score": 26.86828101423543
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 1.2325164356394527
      },
      "insurance_claims": {
       "accuracy": 0.2,
       "cc": -7.320823604982024,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60,
        "score": 21.370862518387256
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.2727272727272727,
       "cc": -4.383667035346825,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -33.333333333333336,
        "score": 22.966058137357102
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.4,
       "cc": 7.379115728815964,
       "cc_by_type": {
        "choice": 53.48837209302325,
        "noul": -33.333333333333336,
        "score": 1.9823084267579727
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 7.379115728815964
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -11.02312325778522,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": 31.4467592589024
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.4074074074074074,
       "cc": -0.8936158432810407,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -60,
        "score": 3.384411587239411
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.7327188940092166,
       "cc": 57.278820382386535,
       "cc_by_type": {
        "choice": 68.44155844155844,
        "noul": 63.63636363636365,
        "score": 39.75853906923753
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 57.278820382386535
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 0.3368320903624114,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 47.07575538816977
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0.3368320903624114
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 50.34599830613678,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 34.41775350708257
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.34599830613678
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -12.550772402850038,
       "cc_by_type": {
        "choice": 26.10837438423645,
        "noul": -100,
        "score": 36.23930840721344
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 5.1704027427711745,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 21.308309677588888
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 5.1704027427711745
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 64.67395698692253,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50,
        "score": 54.6980275443975
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 64.67395698692253
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.8126350640683,
      81.09078920907885
     ],
     "I": [
      26.503639969140373,
      36.23806609472571
     ],
     "capability": [
      50.92622576783774,
      58.04236984861399
     ],
     "composite": {
      "A": [
       12.825119517917724,
       27.43337606642157
      ],
      "B": [
       11.00758497617656,
       24.829735634421777
      ],
      "C": [
       8.906332998553973,
       19.05095560168164
      ]
     },
     "seed": 71
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 78.35523487828182,
     "capability": 55.57071252242379,
     "intelligence": 32.78619016656576
    },
    "per_type": {
     "choice": {
      "accuracy": 0.656,
      "calibration": 74.85231450669417,
      "cc": 55.567462063950714,
      "cc_by_type": {
       "choice": 55.567462063950714
      },
      "cc_tiered": 56.70821112678736,
      "cc_tiered_open": 59.623482293580714,
      "cc_tiered_sealed": 53.79293995999401,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 55.567462063950714
     },
     "noul": {
      "accuracy": 0.5253333333333333,
      "calibration": 80.73005292969255,
      "cc": 5.066666666666664,
      "cc_by_type": {
       "noul": 5.066666666666664
      },
      "cc_tiered": 13.141476880607318,
      "cc_tiered_open": 26.282953761214635,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 5.066666666666664
     },
     "score": {
      "accuracy": 0.488,
      "calibration": 79.48333719845877,
      "cc": 26.316988637484396,
      "cc_by_type": {
       "score": 26.316988637484396
      },
      "cc_tiered": 28.508882492302604,
      "cc_tiered_open": 35.138010794642604,
      "cc_tiered_sealed": 21.879754189962604,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 26.316988637484396
     }
    },
    "raw_on_A": {
     "calibration": 76.9738349679489,
     "capability": 52.86740104845798,
     "intelligence": 28.760967128967057
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 19.49095662175169,
    "Emphasis on Accuracy 60:20:20": 16.457072443157895,
    "Emphasis on Cost 20:20:60": 19.611190631498353,
    "Emphasis on Speed 20:60:20": 23.717988427725025,
    "Intelligence only": 13.341942508299013
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Standard Thinking",
   "axes": {
    "calibration": 77.72284683250058,
    "cost": 48.64218899335634,
    "intelligence": 34.357350203953175,
    "speed": 93.10096025009605
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12117664234817307,
      "ece_score": 75.76467153036539,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 75.76467153036539,
      "score_before_time_limit": 75.76467153036539,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3297587131367292,
      "brier": 0.16945421980960473,
      "ece": 0.10146864036589111,
      "n": 373,
      "score": 79.70627192682178,
      "score_before_time_limit": 79.70627192682178,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8870967741935484,
      "interval90_width": 0.41084229390681,
      "n": 372,
      "nrps": 0.12225177629429217,
      "score": 77.6975970403146,
      "score_before_time_limit": 77.6975970403146,
      "time_limit_failed_share": 0,
      "top_ece": 0.161898141449708
     }
    },
    "score": 77.72284683250058
   },
   "capability": 56.04009851822688,
   "composite_ci95": {
    "A": [
     15.25845206491497,
     31.49102646212039
    ],
    "B": [
     13.201047287610493,
     28.670835708654444
    ],
    "C": [
     10.596147267302063,
     21.868768376472495
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), base-model market reference, the same estimate as the live row standard_one_3b_sh: the model card names base_model StandardThinking/StandardOne-3B, built on mistralai/Ministral-3-3B-Instruct-2512. Neither Standard One checkpoint is listed on OpenRouter; the exact underlying base is priced as OpenRouter mistralai/ministral-3b-2512, list price USD 0.10/M input and USD 0.10/M output (the same OpenRouter document the live sibling cites, read 2026-10-12T00:14Z, source/public-docs/OPENROUTER-MINISTRAL-3-PRICES.json; the frozen module catalogue carries the same 0.10/0.10). Self-hosted weights, no bookable tariff of the measured deployment; the author's stated hosted list price (USD 0.038/M input, standardthinking.ai/pricing) is below this floor, so the base-model reference price applies as a floor. Note on the module: the frozen pricing_v15 module (sha256 aa2afcd5a58d0c4cc0e3d6867d590c70b6ac58e971410c27faf9628f97bcf8b7, re-frozen 22 Oct 2026) has no explicit listing decision for this base string (reference() raises), so the price is the registry value 0.10/0.10 carried over from the live entry and verified against that snapshot, not a module lookup. Estimated, not charged, not a quote from the author or any host. Tokens as reported by the usage block of the author's own server: input_tokens = 764,986 on the 1,485 answered items, output_tokens = 0 on every row (the schema head generates no text; priced at the reference output price). Our offline count through a first-party transcription of the author's renderer and the pinned tokenizer predicted the server's numbers EXACTLY, item by item and in total (764,986 offline = 764,986 native), and equals the 12 Oct receipt of the v2 row (same tokenizer, template and item set). The 15 refused items (HTTP 422, over the author's 32,768-token context) count 0 tokens.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.05099906666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.05151420875420875
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Standard One 3B SH v4 (Standard Thinking)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.519/h. The authors' reference server code/serve_head.py (127.0.0.1:30171, bf16, torch SDPA, no prefix cache) runs in ONE docker container with network none, driven one request at a time over loopback on the VM by the unchanged pinned pod_driver_v16 (frozen runtime modules sha256-pinned in pre-registration Amendment A1). Requests over the documented 32,768-token context answer HTTP 422 (refused, not truncated). The standard self-hosted x2 + 0.15 s latency adjustment applies. Network cut proven from inside the container before any item hash or model load (loopback only, no default route, DNS and TCP fail); read-only code, read-only model and input, empty credential variables (names-only proof), HF offline flags. Measured 23 Oct 2026 (VM 07:12-07:19Z; measurement 66 s, total 359 s; scored run 46 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, re-checked inside the container). The input was shredded from the node and its absence verified, and the VM was removed by exact id afterwards (teardown confirmed 07:19Z).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 42.78829013651281,
    "I_sealed": 25.92641027139353,
    "base": 34.357350203953175,
    "excess": 7.022353094763574,
    "gap": 16.861879865119285,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 61.56799120865807,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 58.47750865051904,
       "judge": 52.537903757415954,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 30.90614216701173,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": -9.999999999999998,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 35.89073703386864,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 44.231557035532845,
       "hard": 37.263336613520615,
       "judge": 44.97516664432012,
       "standard": 15.348483458055373
      }
     },
     "sealed|choice": {
      "cc": 53.65186478422669,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 73.3163913595934,
       "hard": 42.75471261709262,
       "judge": 58.570029382957905,
       "standard": 58.23665893271462
      }
     },
     "sealed|noul": {
      "cc": 2.2708017664270868,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 24.13793103448276,
       "hard": 1.8181818181818077,
       "judge": -13.846153846153841,
       "standard": 16.417910447761198
      }
     },
     "sealed|score": {
      "cc": 21.85656426352682,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 53.598763156448584,
       "hard": 11.93884335357016,
       "judge": 13.357841341148124,
       "standard": 38.56899102054731
      }
     }
    }
   },
   "jevbench_score": 24.395574588088643,
   "key": "standard_one_3b_sh_v4",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 (merged weights, schema head and reference server; base Ministral 3 3B Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "StandardThinking/StandardOne-3B-SH@7f56f73be882bc3fb2b69abe609bab1de8297d77",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.84,
    "decisive": 250,
    "decisive_rate": 0.6702412868632708,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 179,
   "ranked": true,
   "ranks": {
    "A": 179,
    "B": 178,
    "C": 181,
    "capability": 194
   },
   "repo": "https://huggingface.co/StandardThinking/StandardOne-3B-SH",
   "scores": {
    "A": 24.395574588088643,
    "B": 21.82489051004517,
    "C": 16.941371241728227
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21108300800002552,
    "p50_s_raw": 0.030541504000012765,
    "p95_s_adjusted": 0.23198008359992173,
    "p95_s_raw": 0.040990041799960866
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN REFERENCE SERVER, run unmodified, on NEW WEIGHTS: StandardThinking/StandardOne-3B-SH at 7f56f73be882bc3fb2b69abe609bab1de8297d77 (tag v4 = main HEAD on 22 and 23 Oct; MODEL_MANIFEST.json sha256 3071f802ef11e092dfafdd72c49a616b8a288c7a0791e0966f6f2fd55021a863, 26 files, 7,976,303,844 weight bytes, 8 chart PNGs excluded and never read; 5 LFS oids and 21 git-blob oids checked, the author's SHA256SUMS agrees on all 24 non-PNG entries; all 26 files re-verified on the VM after download). Command (the card's Quick-start, all defaults): python3 /code/code/serve_head.py --snapshot /models/StandardOne-3B-SH --head /models/StandardOne-3B-SH/head --port 30171 --served-model-name standardthinking/standard-one-3b-sh (batch 8 requests / 32,768 tokens / 5 ms wait, bf16 backbone, torch SDPA, no cache). The 18 files executed or read at inference (LICENSE, SYSTEM_PROMPT.txt, chat_template.jinja, code/{head,model,render,scoring,serve_head}.py, code/requirements.txt, config.json, generation_config.json, model.safetensors.index.json, params.json, processor_config.json, special_tokens_map.json, tekken.json, tokenizer.json, tokenizer_config.json) are byte-identical (sha256) to the v2 route that was security-reviewed (PR on file / PR on file) and measured on 12 Oct 2026; only the two backbone weight shards, head/schema_head.safetensors and head/schema_head_config.json differ (the config diff is two provenance keys, step 57 -> 11471 and snapshot -> StandardOne-3B-SH@v3, which the server never reads: scoring.py:162-170). The source changes and runner were reviewed before this run; the unchanged serving code retained its earlier clearance. Image: pytorch/pytorch:2.11.0-cuda12.8-cudnn9-runtime@sha256:eee11b3b3872a8c838e35ef48f08b2d5def2080902c7f666831310ca1a0ef2be plus the hash-pinned runtime supplement (19 wheels), built offline on the VM (image sha256:6d8e3c0234ba9923d2c59b85c6b046e6438a34dc1d1504a0218f9a3d00e59f12; the build is not bit-reproducible, the 12 Oct image was 28f68ce6...); transformers 5.12.1, peft 0.21.0, tokenizers 0.22.2, torch 2.11.0, Python 3.12. Container read-only, network none, model and code read-only. Weights: Apache-2.0, public, not gated, tensors in safetensors only (no pickle). FIRST MEASUREMENT OF THIS VERSION. NEW VERSION OF AN EXISTING MODEL (Author submission, free queue, filed by the authors as a follow-up to, not a replacement of, the live row standard_one_3b_sh, which measured the v2 weights at revision bead20e94b7c96045a19dc4d43af48a66c8911ea via paid evaluation on 12 Oct 2026). The two serving versions are reported separately. AUTHOR'S OWN SERVER, on our hardware: the live row ran on a Lium H100; this row runs on one RTX PRO 6000 (bf16 SDPA numerics can differ across GPUs by tiny amounts), so the two rows differ in hardware as well as in weights. SECOND USE OF THE DRAW: evaluator exposure and device ruling admitted this as a same-author second use of the frozen draw for the key family standard_one_3b_sh (first use: the 12 Oct v2 row), with this disclosure; any further StandardThinking row goes to the next draw. 10 s RULE: max measured decision latency 0.116 s (p50 0.031 s, p95 0.041 s, p99 0.102 s); 0 decisions over 10 s. The 15 refused items are the 15 longest (76k-82k tokens), refused with HTTP 422 `too_long` at positions 65, 169, 474, 482, 515, 573, 580, 729, 833, 899, 951, 1057, 1116, 1198, 1418 exactly as pre-registered and identical to the 12 Oct set; they are scored wrong, never truncated. DETERMINISM: first 50 positions run twice, 2/50 bit-identical, 0 argmax flips, max probability difference 9.2e-08. AUTHOR EXPOSURE AND DIRECTION (author's own release-manifest.json, self-reported, not verified): the v4 head was trained from the unsubmitted v3 (head config snapshot StandardOne-3B-SH@v3); between v2 and v4 the author reports its internal `jevbench-original` set falling 98.61 -> 95.83 and `hardproxy` 41.67 -> 38.0 while `internal-eval-v1` rises 84.33 -> 86.33 and `-v2-realistic` 88.33 -> 90.17. The author's training data and overlap screening for v3/v4 were not independently audited by us, and we ran no overlap check against the public 231. OFFLINE COUNTER: a first-party transcription (no author import) that reproduced the 12 Oct native counts exactly; it does not transcribe the span-coverage checks; the exact sealed token match with the 12 Oct receipt establishes the accounting on this draw, not full renderer equivalence. No outside request for this model, and no score, rank or expectation was communicated to the submitter. PREDECESSOR: standard_one_3b_sh (/jev-models/standard_one_3b_sh); both measurements remain separate rows.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6363636363636364,
       "cc": 30.43862198739271,
       "cc_by_type": {
        "choice": 50,
        "noul": 50,
        "score": -8.684134037821867
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 30.43862198739271
      },
      "calibration_gen": {
       "accuracy": 0.6435643564356436,
       "cc": 38.96843819494644,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 43.99999999999999,
        "score": 20.6253570026548
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 38.96843819494644
      },
      "composite_gen": {
       "accuracy": 0.5643564356435643,
       "cc": 24.719858196839443,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -11.111111111111116,
        "score": 17.791010904881468
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 24.719858196839443
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 54.95165466462041,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 44.854963993861254
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 54.95165466462041
      },
      "jaggedness": {
       "accuracy": 0.4388888888888889,
       "cc": 10.357653869679638,
       "cc_by_type": {
        "choice": 27.81520692974015,
        "noul": -35.13513513513513,
        "score": 38.39288981443389
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 10.357653869679638
      },
      "llm_router": {
       "accuracy": 0.7284345047923323,
       "cc": 56.86247915081065,
       "cc_by_type": {
        "choice": 66.71656247248393,
        "noul": 60.82474226804124,
        "score": 43.04613271190677
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 56.86247915081065
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3333333333333333,
       "cc": -31.17577443919728,
       "cc_by_type": {
        "choice": 1.0309278350515454,
        "noul": -75,
        "score": -19.55825115264338
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6,
       "cc": 29.820965381161074,
       "cc_by_type": {
        "choice": 60.873818063254,
        "noul": -2.777777777777779,
        "score": 31.366855858007003
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 29.820965381161074
      },
      "negation_pair_gen": {
       "accuracy": 0.6395348837209303,
       "cc": 21.33088783112437,
       "cc_by_type": {
        "choice": 80,
        "noul": -19.999999999999996,
        "score": 3.9926634933731164
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 21.33088783112437
      },
      "option_order": {
       "accuracy": 0.5625,
       "cc": 44.074567243675105,
       "cc_by_type": {
        "choice": 44.074567243675105
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 44.074567243675105
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 75.75773444552337,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 100,
        "score": 63.30923937260613
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 75.75773444552337
      },
      "usecase": {
       "accuracy": 0.4954545454545455,
       "cc": 18.505383917285155,
       "cc_by_type": {
        "choice": 54.65686274509807,
        "noul": -16.666666666666664,
        "score": 17.525955673424054
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 18.505383917285155
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.3125,
       "cc": -16.223038433749565,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": 50,
        "score": -84.3834010155344
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 40.03490559353603,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": -13.228616552725248
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 40.03490559353603
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 68.64915139458252,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 54.841259493482085
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 68.64915139458252
      },
      "el": {
       "accuracy": 0.4,
       "cc": 19.79738156682247,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 93.09791751238548
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 19.79738156682247
      },
      "en": {
       "accuracy": 0.5899218071242398,
       "cc": 31.93655338456402,
       "cc_by_type": {
        "choice": 54.30858788640606,
        "noul": 14.388489208633093,
        "score": 27.112583058652916
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 31.93655338456402
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 46.78575129260497,
       "cc_by_type": {
        "choice": 70.63621533442088,
        "noul": 50,
        "score": 19.721038543394044
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 46.78575129260497
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -7.626126671344963,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": 75.48227572367003
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 43.697632308903025,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 52.26936751494436
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 43.697632308903025
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 25.716200068649442,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": -25,
        "score": 35.668153278574025
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 25.716200068649442
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 60.9707899465536,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": 100,
        "score": 10.971606004898549
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 60.9707899465536
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 23.91155193522305,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 0,
        "score": 14.625839284620701
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 23.91155193522305
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -15.139117321498192,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": -0.9729075200501258
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 68.65378847904408,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 89.29469877046562
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 68.65378847904408
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 18.275963913872197,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 14.135251048975883
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 18.275963913872197
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5641025641025641,
       "cc": 26.945577097442794,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -19.999999999999996,
        "score": 31.306473145904345
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 26.945577097442794
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 62.57546899861372,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 100,
        "score": 30.154220784880657
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 62.57546899861372
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 10.245504472118656,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 52.864172990824045
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 10.245504472118656
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": -4.064699344436785,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": -68.71583716374514
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 67.03729233509661,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 65.62800603754792
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.03729233509661
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 18.87224921870604,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": 33.33333333333333,
        "score": -51.29285686365588
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 18.87224921870604
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6573333333333333,
       "cc": 55.739681203237716,
       "cc_by_type": {
        "choice": 55.739681203237716
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 55.739681203237716
      },
      "noul": {
       "accuracy": 0.56,
       "cc": 12.00000000000001,
       "cc_by_type": {
        "noul": 12.00000000000001
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 12.00000000000001
      },
      "score": {
       "accuracy": 0.5066666666666667,
       "cc": 27.031098893672045,
       "cc_by_type": {
        "score": 27.031098893672045
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 27.031098893672045
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3181818181818182,
       "cc": -11.271742464625499,
       "cc_by_type": {
        "choice": 68,
        "noul": -75,
        "score": -26.815227393876494
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 7.5198762513336215,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": -14.017320866427019
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 7.5198762513336215
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 49.10493423741544,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 29.83512791549836
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 49.10493423741544
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -44.25518595295127,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -50.89421282961406
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 14.797957491146107,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 6.712713053148455
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 14.797957491146107
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -21.042436730888507,
       "cc_by_type": {
        "choice": 49.685534591194966,
        "noul": -33.333333333333336,
        "score": -79.47951145052716
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 3.4881564022506595,
       "cc_by_type": {
        "choice": 60.784313725490186,
        "noul": -71.42857142857143,
        "score": 21.10872690983322
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 3.4881564022506595
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -2.2430386640931927,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50,
        "score": 16.441615715037493
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": -11.621822353601146,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": -12.365467060803436
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.36363636363636365,
       "cc": 6.876774725855042,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": 0,
        "score": 23.414050087629366
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 6.876774725855042
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 28.644604238960103,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": -33.333333333333336,
        "score": 42.52296000370202
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 28.644604238960103
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 11.417587720502413,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": -34.564441139568025
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 11.417587720502413
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 7.739621289484592,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -60,
        "score": 29.28412298553631
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 7.739621289484592
      },
      "model_routing": {
       "accuracy": 0.7235023041474654,
       "cc": 54.23699976619281,
       "cc_by_type": {
        "choice": 73.11688311688313,
        "noul": 60.606060606060595,
        "score": 28.988055575634696
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 54.23699976619281
      },
      "moderation_trust_safety": {
       "accuracy": 0.3888888888888889,
       "cc": -5.564829778188596,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 29.37076978251675
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 47.02903163973576,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 24.46685350787955
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 47.02903163973576
      },
      "risk_assessment": {
       "accuracy": 0.4444444444444444,
       "cc": -8.643300724068723,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -100,
        "score": 18.405073197251966
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.2608695652173913,
       "cc": -20.59884799463212,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": -66.66666666666667,
        "score": -8.173355578099262
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.7916666666666666,
       "cc": 63.30212811247312,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 50,
        "score": 61.25869750467913
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 63.30212811247312
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.01156479992845,
      80.83105889189729
     ],
     "I": [
      28.310092387743147,
      38.15335637133141
     ],
     "capability": [
      51.20193899458882,
      58.72712563492504
     ],
     "composite": {
      "A": [
       15.25845206491497,
       31.49102646212039
      ],
      "B": [
       13.201047287610493,
       28.670835708654444
      ],
      "C": [
       10.596147267302063,
       21.868768376472495
      ]
     },
     "seed": 73
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.72284683250058,
     "capability": 56.04009851822688,
     "intelligence": 34.357350203953175
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6573333333333333,
      "calibration": 75.76467153036539,
      "cc": 55.739681203237716,
      "cc_by_type": {
       "choice": 55.739681203237716
      },
      "cc_tiered": 57.60992799644238,
      "cc_tiered_open": 61.56799120865807,
      "cc_tiered_sealed": 53.65186478422669,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 55.739681203237716
     },
     "noul": {
      "accuracy": 0.56,
      "calibration": 79.70627192682178,
      "cc": 12.00000000000001,
      "cc_by_type": {
       "noul": 12.00000000000001
      },
      "cc_tiered": 16.588471966719407,
      "cc_tiered_open": 30.90614216701173,
      "cc_tiered_sealed": 2.2708017664270868,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 12.00000000000001
     },
     "score": {
      "accuracy": 0.5066666666666667,
      "calibration": 77.6975970403146,
      "cc": 27.031098893672045,
      "cc_by_type": {
       "score": 27.031098893672045
      },
      "cc_tiered": 28.87365064869773,
      "cc_tiered_open": 35.89073703386864,
      "cc_tiered_sealed": 21.85656426352682,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.031098893672045
     }
    },
    "raw_on_A": {
     "calibration": 83.50326227116736,
     "capability": 58.44228421245579,
     "intelligence": 33.38130615374423
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 22.19385254275724,
    "Emphasis on Accuracy 60:20:20": 18.836883828876903,
    "Emphasis on Cost 20:20:60": 22.008806459889172,
    "Emphasis on Speed 20:60:20": 27.28596975238759,
    "Intelligence only": 15.353422081196133
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Standard Thinking",
   "axes": {
    "calibration": 85.49831885164681,
    "cost": 43.21124696592469,
    "intelligence": 34.464905061466695,
    "speed": 84.17488059602204
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07091377459977057,
      "ece_score": 85.8172450800459,
      "mean_tvd": null,
      "n": 739,
      "n_ece_hard": 317,
      "n_tvd": 0,
      "score": 85.8172450800459,
      "score_before_time_limit": 85.8172450800459,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.39010989010989006,
      "brier": 0.15382831780922446,
      "ece": 0.0674352050569083,
      "n": 364,
      "score": 86.51295898861834,
      "score_before_time_limit": 86.51295898861834,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9320652173913043,
      "interval90_width": 0.5106431159420289,
      "n": 368,
      "nrps": 0.11464419484173276,
      "score": 84.1647524862762,
      "score_before_time_limit": 84.1647524862762,
      "time_limit_failed_share": 0,
      "top_ece": 0.10103037771637155
     }
    },
    "score": 85.49831885164681
   },
   "capability": 59.981611956556755,
   "composite_ci95": {
    "A": [
     12.630725624836908,
     23.78332080006789
    ],
    "B": [
     11.12675433052624,
     21.913417722125814
    ],
    "C": [
     8.771337239470075,
     16.516195000047144
    ]
   },
   "cost": {
    "basis": "v1.5-M2 base-model reference: model card base_model StandardThinking/StandardOne-8B, built on mistralai/Ministral-3-8B-Instruct-2512 (not listed on OpenRouter as Standard One). OpenRouter mistralai/ministral-8b-2512 list price USD 0.15/M input and USD 0.15/M output, read 2026-10-10T08:43Z from https://openrouter.ai/api/v1/models. Schema head generates no text: 0 output tokens.",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0781553
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Standard One 8B SH (Standard Thinking)",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100 80GB), offline read-only container (--network none); author's reference server code/serve_head.py, unchanged defaults",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 42.65983609411336,
    "I_sealed": 26.269974028820027,
    "base": 34.464905061466695,
    "excess": 6.316617483953619,
    "gap": 16.389862065293332,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 64.76188322786868,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432432,
       "hard": 56.59012268008808,
       "judge": 64.40342781806197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 26.588681849551417,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 36.628943204920006,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 20.91980977072071,
       "hard": 40.258980287858556,
       "judge": 46.927298157653716,
       "standard": 21.775903327041945
      }
     },
     "sealed|choice": {
      "cc": 53.894778364676796,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 60.57983105641882,
       "hard": 50.5331427943278,
       "judge": 51.60062909035564,
       "standard": 60.716747070985534
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 12.82051282051282,
       "hard": -9.090909090909093,
       "judge": 11.111111111111116,
       "standard": -8.771929824561408
      }
     },
     "sealed|score": {
      "cc": 24.915143721783284,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 48.44464458742374,
       "hard": 10.686057111566038,
       "judge": 28.029772063789803,
       "standard": 36.93662399638777
      }
     }
    }
   },
   "jevbench_score": 18.74307320095725,
   "key": "standard_one_8b_sh",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (merged weights, schema head and reference server; base Ministral 3 8B Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "StandardThinking/StandardOne-8B-SH@25e5739bd9635b54f925cf9ba2daa2ec42afd19e",
   "noul_decisive": {
    "acc_among_decisive": 0.8828828828828829,
    "decisive": 222,
    "decisive_rate": 0.6098901098901099,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.49831885164681,
     "cost": 43.21124696592469,
     "intelligence": 34.464905061466695,
     "speed": 84.28099014807135
    },
    "board_release": "v1.6.43",
    "capability": 59.981611956556755,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      12.633047380555162,
      23.788181701570505
     ],
     "B": [
      11.128195660262914,
      21.916709714788084
     ],
     "C": [
      8.772949569829972,
      16.519570626090626
     ]
    },
    "jevbench_score": 18.74677558938401,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 103,
    "rank": 138,
    "ranks": {
     "A": 138,
     "B": 141,
     "C": 144,
     "capability": 132
    },
    "scores": {
     "A": 18.74677558938401,
     "B": 16.94153091767041,
     "C": 13.018594159294452
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.27213579495437445,
     "p50_s_raw": 0.061067897477187216,
     "p95_s_adjusted": 1.3712458158901424,
     "p95_s_raw": 0.6106229079450712
    },
    "views": {
     "Balanced 33:33:33": 16.62869989264526,
     "Emphasis on Accuracy 60:20:20": 14.53757347040618,
     "Emphasis on Cost 20:20:60": 16.08558260894422,
     "Emphasis on Speed 20:60:20": 20.21989737661098,
     "Intelligence only": 12.230517698051182
    }
   },
   "rank": 189,
   "ranked": true,
   "ranks": {
    "A": 189,
    "B": 188,
    "C": 190,
    "capability": 165
   },
   "repo": "https://huggingface.co/StandardThinking/StandardOne-8B-SH",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 18.74307320095725,
    "B": 16.939111850803307,
    "C": 13.016023056220313
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.28489439501427116,
    "p50_s_raw": 0.06744719750713557,
    "p95_s_adjusted": 1.3422333808033726,
    "p95_s_raw": 0.5961166904016864
   },
   "status": {
    "answered_ok": 1471,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.896551724137931,
       "cc": 76.24972247673712,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 45.41583409687805
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 76.24972247673712
      },
      "calibration_gen": {
       "accuracy": 0.6339285714285714,
       "cc": 40.6073873879211,
       "cc_by_type": {
        "choice": 60.40372670807453,
        "noul": 30.76923076923077,
        "score": 30.64920468645799
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 40.6073873879211
      },
      "composite_gen": {
       "accuracy": 0.6428571428571429,
       "cc": 42.991452161771214,
       "cc_by_type": {
        "choice": 70.37037037037037,
        "noul": 28.57142857142858,
        "score": 30.032557543514706
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 42.991452161771214
      },
      "fairness": {
       "accuracy": 0.9629629629629629,
       "cc": 88.7254774240554,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 66.17643227216622
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 88.7254774240554
      },
      "jaggedness": {
       "accuracy": 0.4702702702702703,
       "cc": 8.921712067552678,
       "cc_by_type": {
        "choice": 34.69198703103288,
        "noul": -53.84615384615385,
        "score": 45.919303017779
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 8.921712067552678
      },
      "llm_router": {
       "accuracy": 0.6477272727272727,
       "cc": 47.79185607418858,
       "cc_by_type": {
        "choice": 57.268927078495125,
        "noul": 26.027397260273965,
        "score": 60.07924388379662
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 47.79185607418858
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3225806451612903,
       "cc": -19.288646333408973,
       "cc_by_type": {
        "choice": 26.153846153846153,
        "noul": -75,
        "score": -9.019785154073068
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5873015873015873,
       "cc": 28.891423678941237,
       "cc_by_type": {
        "choice": 56.112852664576806,
        "noul": 8.045977011494255,
        "score": 22.515441360752643
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 28.891423678941237
      },
      "negation_pair_gen": {
       "accuracy": 0.5357142857142857,
       "cc": -27.351155376155678,
       "cc_by_type": {
        "choice": 76.19047619047619,
        "noul": -50,
        "score": -108.24394231894323
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.9122807017543859,
       "cc": 88.90122086570477,
       "cc_by_type": {
        "choice": 88.90122086570477
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 88.90122086570477
      },
      "perturb_pair": {
       "accuracy": 0.8333333333333334,
       "cc": 73.07758294752922,
       "cc_by_type": {
        "choice": 76.4089121887287,
        "noul": 71.42857142857142,
        "score": 71.39526522528755
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 73.07758294752922
      },
      "usecase": {
       "accuracy": 0.4978902953586498,
       "cc": 20.903453186109534,
       "cc_by_type": {
        "choice": 45.65426864521104,
        "noul": 1.7543859649122862,
        "score": 15.301704948205275
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 20.903453186109534
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.2727272727272727,
       "cc": -41.39521348113611,
       "cc_by_type": {
        "choice": -6.666666666666665,
        "noul": 0,
        "score": -117.51897377674166
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.375,
       "cc": -4.802259887005647,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.5833333333333334,
       "cc": 23.301107918011525,
       "cc_by_type": {
        "choice": 28.994082840236683,
        "noul": -19.999999999999996,
        "score": 60.90924091379788
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 23.301107918011525
      },
      "de": {
       "accuracy": 0.6444444444444445,
       "cc": 44.56362261696139,
       "cc_by_type": {
        "choice": 72.84248642124321,
        "noul": 55.55555555555556,
        "score": 5.292825874085382
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 44.56362261696139
      },
      "el": {
       "accuracy": 0.5,
       "cc": -22.37065677373447,
       "cc_by_type": {
        "choice": 63.855421686746986,
        "noul": -60,
        "score": -70.96739200795041
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.5978552278820375,
       "cc": 30.293789733525,
       "cc_by_type": {
        "choice": 56.70218732473357,
        "noul": 3.007518796992481,
        "score": 31.171663078848944
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 30.293789733525
      },
      "es": {
       "accuracy": 0.7142857142857143,
       "cc": 43.83935759486882,
       "cc_by_type": {
        "choice": 100,
        "noul": 45.45454545454546,
        "score": -13.936472669939004
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 43.83935759486882
      },
      "fi": {
       "accuracy": 0.2222222222222222,
       "cc": -42.25352112676057,
       "cc_by_type": {
        "choice": 15.49295774647887,
        "noul": -100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.4666666666666667,
       "cc": -0.26931923616449654,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": -100,
        "score": 49.96778277105517
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.4,
       "cc": 3.9605189196118,
       "cc_by_type": {
        "choice": 25.465838509316765,
        "noul": -19.999999999999996,
        "score": 6.415718249518632
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 3.9605189196118
      },
      "id": {
       "accuracy": 0.5,
       "cc": -16.12903225806452,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7647058823529411,
       "cc": 42.698737038013526,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 0,
        "score": 28.09621111404058
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 42.698737038013526
      },
      "ja": {
       "accuracy": 0.5,
       "cc": 9.04085560182647,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": -42.85714285714286,
        "score": 35.02036006912633
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 9.04085560182647
      },
      "ko": {
       "accuracy": 0.6363636363636364,
       "cc": 51.84513737928095,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": -11.677702616255514
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 51.84513737928095
      },
      "mixed": {
       "accuracy": 1,
       "cc": 95.1901995158086,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 85.57059854742577
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 95.1901995158086
      },
      "nl": {
       "accuracy": 0.46153846153846156,
       "cc": -6.51647728738407,
       "cc_by_type": {
        "choice": 49.860724233983284,
        "noul": 0,
        "score": -69.4101560961355
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.42857142857142855,
       "cc": 17.486338797814206,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 17.486338797814206
      },
      "pl": {
       "accuracy": 0.575,
       "cc": 41.28299201238961,
       "cc_by_type": {
        "choice": 76.38828159160472,
        "noul": 19.999999999999996,
        "score": 27.460694445564116
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 41.28299201238961
      },
      "pt": {
       "accuracy": 0.8333333333333334,
       "cc": 76.5780001477811,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": 100,
        "score": 47.82445270464984
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 76.5780001477811
      },
      "sv": {
       "accuracy": 0.5,
       "cc": 6.320693289140117,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": 50,
        "score": -42.14903124369076
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 6.320693289140117
      },
      "tr": {
       "accuracy": 0.6363636363636364,
       "cc": 22.124025198251957,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 33.33333333333333,
        "score": -40.64546826489326
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 22.124025198251957
      },
      "uk": {
       "accuracy": 0.5454545454545454,
       "cc": -5.12371977540209,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100,
        "score": 49.84623197814155
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.875,
       "cc": 62.994622040580985,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 33.33333333333333,
        "score": 55.65053278840959
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 62.994622040580985
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6653333333333333,
       "cc": 56.64190706512352,
       "cc_by_type": {
        "choice": 56.64190706512352
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 56.64190706512352
      },
      "noul": {
       "accuracy": 0.5226666666666666,
       "cc": 4.533333333333323,
       "cc_by_type": {
        "noul": 4.533333333333323
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 4.533333333333323
      },
      "score": {
       "accuracy": 0.5173333333333333,
       "cc": 27.98616472792205,
       "cc_by_type": {
        "score": 27.98616472792205
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 27.98616472792205
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5833333333333334,
       "cc": 2.1721868800764432,
       "cc_by_type": {
        "choice": 50.57660626029654,
        "noul": -60,
        "score": 15.939954379932786
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 2.1721868800764432
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": 4.537943129820239,
       "cc_by_type": {
        "choice": 1.8003273322422266,
        "noul": 60.00000000000001,
        "score": -48.186497942781514
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 4.537943129820239
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 18.203665219669762,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -25,
        "score": 44.12712469126735
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 18.203665219669762
      },
      "demand_forecasting": {
       "accuracy": 0.3076923076923077,
       "cc": -15.30308720258734,
       "cc_by_type": {
        "choice": 27.710843373493972,
        "noul": -50,
        "score": -23.620104981255995
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5555555555555556,
       "cc": 31.8743625129997,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 19.999999999999996,
        "score": 31.523708657011518
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 31.8743625129997
      },
      "feature_extraction": {
       "accuracy": 0.5172413793103449,
       "cc": 16.864472729203204,
       "cc_by_type": {
        "choice": 31.623931623931618,
        "noul": -19.999999999999996,
        "score": 38.96948656367799
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 16.864472729203204
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 15.494473495061596,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0,
        "score": 11.700811789532606
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 15.494473495061596
      },
      "gaming": {
       "accuracy": 0.3793103448275862,
       "cc": -6.714417440674549,
       "cc_by_type": {
        "choice": 40.59405940594059,
        "noul": -42.85714285714286,
        "score": -17.88016887082138
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.5,
       "cc": 28.798707116861394,
       "cc_by_type": {
        "choice": 52.38095238095238,
        "noul": -60,
        "score": 94.0151689696318
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 28.798707116861394
      },
      "knowledge_graphs": {
       "accuracy": 0.391304347826087,
       "cc": -0.31602635093334186,
       "cc_by_type": {
        "choice": 35.87786259541985,
        "noul": -60,
        "score": 23.174058351780126
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.36,
       "cc": 1.8570013967353607,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -50,
        "score": -28.168833208167896
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 1.8570013967353607
      },
      "legal_compliance": {
       "accuracy": 0.3076923076923077,
       "cc": -26.63894890640322,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": -14.343076227406382
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.7241379310344828,
       "cc": 51.33693366193592,
       "cc_by_type": {
        "choice": 72.4770642201835,
        "noul": 50,
        "score": 31.533736765624287
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 51.33693366193592
      },
      "model_routing": {
       "accuracy": 0.68,
       "cc": 48.65467531366116,
       "cc_by_type": {
        "choice": 75.87778075582953,
        "noul": 36.36363636363635,
        "score": 33.72260882151758
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 48.65467531366116
      },
      "moderation_trust_safety": {
       "accuracy": 0.5625,
       "cc": 25.43524834873031,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 19.999999999999996,
        "score": 19.463639783033038
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 25.43524834873031
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 52.73441114090357,
       "cc_by_type": {
        "choice": 84.92462311557789,
        "noul": 33.33333333333333,
        "score": 39.945276973799494
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 52.73441114090357
      },
      "risk_assessment": {
       "accuracy": 0.23076923076923078,
       "cc": -34.30633768347406,
       "cc_by_type": {
        "choice": 5.660377358490565,
        "noul": -100,
        "score": -8.579390408912758
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.40625,
       "cc": -3.5610027209667976,
       "cc_by_type": {
        "choice": 36.254980079681275,
        "noul": -71.42857142857143,
        "score": 24.490583185989767
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.7272727272727273,
       "cc": 72.3432480209567,
       "cc_by_type": {
        "choice": 85.68019093078759,
        "noul": 100,
        "score": 31.349553132082498
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 72.3432480209567
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.15231941930905,
      86.38112582622072
     ],
     "I": [
      29.324371200107983,
      38.19996868954735
     ],
     "capability": [
      55.13570575242618,
      61.67190307298117
     ],
     "composite": {
      "A": [
       12.630725624836908,
       23.78332080006789
      ],
      "B": [
       11.12675433052624,
       21.913417722125814
      ],
      "C": [
       8.771337239470075,
       16.516195000047144
      ]
     },
     "seed": 28
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.49831885164681,
     "capability": 59.981611956556755,
     "intelligence": 34.464905061466695
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6653333333333333,
      "calibration": 85.8172450800459,
      "cc": 56.64190706512352,
      "cc_by_type": {
       "choice": 56.64190706512352
      },
      "cc_tiered": 59.32833079627274,
      "cc_tiered_open": 64.76188322786868,
      "cc_tiered_sealed": 53.894778364676796,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 56.64190706512352
     },
     "noul": {
      "accuracy": 0.5226666666666666,
      "calibration": 86.51295898861834,
      "cc": 4.533333333333323,
      "cc_by_type": {
       "noul": 4.533333333333323
      },
      "cc_tiered": 13.294340924775709,
      "cc_tiered_open": 26.588681849551417,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 4.533333333333323
     },
     "score": {
      "accuracy": 0.5173333333333333,
      "calibration": 84.1647524862762,
      "cc": 27.98616472792205,
      "cc_by_type": {
       "score": 27.98616472792205
      },
      "cc_tiered": 30.772043463351643,
      "cc_tiered_open": 36.628943204920006,
      "cc_tiered_sealed": 24.915143721783284,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.98616472792205
     }
    },
    "raw_on_A": {
     "calibration": 83.94383086837013,
     "capability": 57.82491809564761,
     "intelligence": 31.706005322925098
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 16.624815988733957,
    "Emphasis on Accuracy 60:20:20": 14.535792178016427,
    "Emphasis on Cost 20:20:60": 16.083401791604228,
    "Emphasis on Speed 20:60:20": 20.209563536903357,
    "Intelligence only": 12.230517698051182
   }
  },
  {
   "adapter_id": "images/standardone-8b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Standard Thinking (myeongho12)",
   "axes": {
    "calibration": 84.01508092906352,
    "cost": 43.283250331904235,
    "intelligence": 32.80426604507879,
    "speed": 92.46998352781172
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.1098197498466637,
      "ece_score": 78.03605003066725,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 78.03605003066725,
      "score_before_time_limit": 78.03605003066725,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.32345013477088946,
      "brier": 0.1399053250455738,
      "ece": 0.07067323668547182,
      "n": 371,
      "score": 85.86535266290564,
      "score_before_time_limit": 85.86535266290564,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.907859078590786,
      "interval90_width": 0.4114724480578141,
      "n": 369,
      "nrps": 0.10288890604189731,
      "score": 88.14384009361767,
      "score_before_time_limit": 88.14384009361767,
      "time_limit_failed_share": 0,
      "top_ece": 0.06711714604287465
     }
    },
    "score": 84.01508092906352
   },
   "capability": 58.40967348707115,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     11.013693486330085,
     24.234025771068143
    ],
    "B": [
     9.565594491333835,
     22.271817937890987
    ],
    "C": [
     7.648398254395891,
     16.829184563241764
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.07772456896551724
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Standard One 8B (Standard Thinking)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 30.33450693574831,
    "I_sealed": 35.27402515440927,
    "base": 32.80426604507879,
    "excess": -7.512905861099208,
    "gap": -4.9395182186609645,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 49.295789751985026,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.2972972972973,
       "hard": 52.815350739226176,
       "judge": 44.62755438365195,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 8.598343685300202,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 13.043478260869556,
       "judge": -30.000000000000004,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 33.109387369959705,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.66883781130674,
       "hard": 20.17576470368565,
       "judge": 52.27846833393313,
       "standard": 20.443286035874163
      }
     },
     "sealed|choice": {
      "cc": 51.12536859237399,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 70.73387037173899,
       "hard": 39.42307692307691,
       "judge": 57.16945996275605,
       "standard": 55.65956398571254
      }
     },
     "sealed|noul": {
      "cc": 17.63077432402283,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 56.00000000000001,
       "hard": -12.621359223300976,
       "judge": 31.578947368421062,
       "standard": 38.0281690140845
      }
     },
     "sealed|score": {
      "cc": 37.06593254683102,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 56.312646632383355,
       "hard": 30.71497074829812,
       "judge": 39.8913069210783,
       "standard": 35.90643753974974
      }
     }
    }
   },
   "jevbench_score": 16.909498058775554,
   "key": "standardone-8b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (jev-adapter server, LoRA and merged weights; base Ministral 3 8B Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.900398406374502,
    "decisive_rate": 0.6765498652291105,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.01508092906352,
     "cost": 43.283250331904235,
     "intelligence": 32.80426604507879,
     "speed": 92.76442849252426
    },
    "board_release": "v1.6.43",
    "capability": 58.40967348707115,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      11.018270368438198,
      24.24546849303509
     ],
     "B": [
      9.568356102409696,
      22.27960999090091
     ],
     "C": [
      7.651576644748748,
      16.837130897941034
     ]
    },
    "jevbench_score": 16.917108299009165,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 112,
    "rank": 149,
    "ranks": {
     "A": 149,
     "B": 149,
     "C": 150,
     "capability": 140
    },
    "scores": {
     "A": 16.917108299009165,
     "B": 15.107984144533011,
     "C": 11.747991874311923
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21347399484366178,
     "p50_s_raw": 0.03173699742183089,
     "p95_s_adjusted": 0.24786326913163065,
     "p95_s_raw": 0.04893163456581533
    },
    "views": {
     "Balanced 33:33:33": 15.03402443438624,
     "Emphasis on Accuracy 60:20:20": 12.868191666930013,
     "Emphasis on Cost 20:20:60": 14.585944618765962,
     "Emphasis on Speed 20:60:20": 18.769748652230398,
     "Intelligence only": 10.581584191343778
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 196,
   "ranked": true,
   "ranks": {
    "A": 196,
    "B": 194,
    "C": 196,
    "capability": 173
   },
   "repo": "https://huggingface.co/StandardThinking/StandardOne-8B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.909498058775554,
    "B": 15.103127851027393,
    "C": 11.742706985260801
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21601051054894924,
    "p50_s_raw": 0.03300525527447462,
    "p95_s_adjusted": 0.2621360577642917,
    "p95_s_raw": 0.056068028882145876
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.9630533878547,
      85.6304653046258
     ],
     "I": [
      27.551137017862857,
      38.18793043954382
     ],
     "capability": [
      53.64856195871698,
      60.83707776399651
     ],
     "composite": {
      "A": [
       11.013693486330085,
       24.234025771068143
      ],
      "B": [
       9.565594491333835,
       22.271817937890987
      ],
      "C": [
       7.648398254395891,
       16.829184563241764
      ]
     },
     "seed": 103
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.01508092906352,
     "capability": 58.40967348707115,
     "intelligence": 32.80426604507879
    },
    "per_type": {
     "choice": {
      "accuracy": 0.62,
      "calibration": 78.03605003066725,
      "cc": 51.40342733722988,
      "cc_by_type": {
       "choice": 51.40342733722988
      },
      "cc_tiered": 50.21057917217951,
      "cc_tiered_open": 49.295789751985026,
      "cc_tiered_sealed": 51.12536859237399,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 51.40342733722988
     },
     "noul": {
      "accuracy": 0.6026666666666667,
      "calibration": 85.86535266290564,
      "cc": 20.53333333333334,
      "cc_by_type": {
       "noul": 20.53333333333334
      },
      "cc_tiered": 13.114559004661515,
      "cc_tiered_open": 8.598343685300202,
      "cc_tiered_sealed": 17.63077432402283,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 20.53333333333334
     },
     "score": {
      "accuracy": 0.552,
      "calibration": 88.14384009361767,
      "cc": 36.63215814216161,
      "cc_by_type": {
       "score": 36.63215814216161
      },
      "cc_tiered": 35.08765995839536,
      "cc_tiered_open": 33.109387369959705,
      "cc_tiered_sealed": 37.06593254683102,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.63215814216161
     }
    },
    "raw_on_A": {
     "calibration": 79.00702916038831,
     "capability": 58.75187777403198,
     "intelligence": 38.49672638767566
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 15.026011359123842,
    "Emphasis on Accuracy 60:20:20": 12.864668383151253,
    "Emphasis on Cost 20:20:60": 14.58141808431248,
    "Emphasis on Speed 20:60:20": 18.74728142290813,
    "Intelligence only": 10.581584191343778
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "StartLux Labs",
   "axes": {
    "calibration": 90.90819924197592,
    "cost": 56.56867594442542,
    "intelligence": 66.69999992075427,
    "speed": 75.15747300434305
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03848208596918053,
      "ece_score": 92.3035828061639,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 91.8112970311977,
      "score_before_time_limit": 92.3035828061639,
      "time_limit_failed_share": 0.005333333333333333
     },
     "noul": {
      "abstention_rate": 0.2278820375335121,
      "brier": 0.06028868195208473,
      "ece": 0.05057905949763694,
      "n": 373,
      "score": 89.64449693220469,
      "score_before_time_limit": 89.88418810047261,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.989247311827957,
      "interval90_width": 0.2383512544802867,
      "n": 372,
      "nrps": 0.041938928585832994,
      "score": 91.2688037625254,
      "score_before_time_limit": 93.004895138443,
      "time_limit_failed_share": 0.018666666666666668,
      "top_ece": 0.04898158432265336
     }
    },
    "score": 90.90819924197592
   },
   "capability": 78.8040995813651,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 737,990 input + 72,063 output tokens over 1,485 decisions = 545.49023569 tokens/decision. GPU-time cost USD 0.0280356611217739/1,000; old list-price cost USD 0.354306262626263/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0280356611217739/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.3507632
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02803566112177389
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 12,
    "n_over_limit_answered_ok": 12,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "StartLux-Decision v1.1 27B AutoThink (StartLux Labs, Qwen3.8-27B fine-tune, gate 0.6)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H200 141 GB (sm_90, driver 580.173.02), Finland (EU). The author's own server on vLLM 0.19.1, offline, loopback, one request at a time.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 67.83827136985272,
    "I_sealed": 65.56172847165581,
    "base": 66.69999992075427,
    "excess": -7.562983872158803,
    "gap": 2.276542898196908,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 94.7183214036337,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 90.56307014784524,
       "judge": 100,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 47.46790890269151,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 65.21739130434783,
       "judge": 30.000000000000004,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 61.328583803232945,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 67.57481560088962,
       "hard": 84.34403756537992,
       "judge": 31.19020107738477,
       "standard": 57.38213446888292
      }
     },
     "sealed|choice": {
      "cc": 88.15105310694662,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 98.09402795425667,
       "hard": 83.34682548860874,
       "judge": 90.30362389813908,
       "standard": 89.55916473317865
      }
     },
     "sealed|noul": {
      "cc": 46.833366324874305,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 58.62068965517242,
       "hard": 39.99999999999999,
       "judge": 38.46153846153846,
       "standard": 67.16417910447761
      }
     },
     "sealed|score": {
      "cc": 61.700765983146525,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 76.7463644799391,
       "hard": 54.79544186930036,
       "judge": 65.8141729534546,
       "standard": 61.81850450698043
      }
     }
    }
   },
   "jevbench_score": 70.20544677571385,
   "key": "startlux-decision-v1.1-27b-autothink",
   "last_measured_on": "2026-10-16",
   "licence": "Weights CC BY-NC 4.0 (non-commercial: commercial use needs a separate licence from StartLux Labs; listing a measured score is evaluation use); inference package Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.25",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "startlux-models/StartLux-Decision-v1.1-27B-AutoThink@dd62d5cd84a483d75bdd548880bbf08bdd3f67a4",
   "noul_decisive": {
    "acc_among_decisive": 0.96875,
    "decisive": 288,
    "decisive_rate": 0.7741935483870968,
    "n": 375,
    "supported": true,
    "valid": 372
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.73088868169317,
     "cost": 23.51863513364647,
     "intelligence": 68.96369650479022,
     "speed": 75.05228443363347
    },
    "board_release": "v1.6.43",
    "capability": 80.34729259324169,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      10.723171777210583,
      10.982898737896896
     ],
     "B": [
      11.285116241749748,
      11.74017776896854
     ],
     "C": [
      10.723171777210583,
      10.982898737896896
     ]
    },
    "jevbench_score": 10.892954017975724,
    "measured_in": "v1.6.25",
    "method": "v1.6",
    "open_board_rank": 129,
    "rank": 168,
    "ranks": {
     "A": 168,
     "B": 162,
     "C": 158,
     "capability": 23
    },
    "scores": {
     "A": 10.892954017975724,
     "B": 11.554064135977844,
     "C": 10.892954017975724
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26122978397179397,
     "p50_s_raw": 0.055614891985896975,
     "p95_s_adjusted": 11.960487055202247,
     "p95_s_raw": 5.905243527601123
    },
    "views": {
     "Balanced 33:33:33": 9.435806388573084,
     "Emphasis on Accuracy 60:20:20": 11.135498515443802,
     "Emphasis on Cost 20:20:60": 7.119526996383961,
     "Emphasis on Speed 20:60:20": 11.405605564551664,
     "Intelligence only": 15.258250914248908
    }
   },
   "rank": 74,
   "ranked": true,
   "ranks": {
    "A": 74,
    "B": 58,
    "C": 58,
    "capability": 42
   },
   "repo": "https://huggingface.co/startlux-models/StartLux-Decision-v1.1-27B-AutoThink",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 70.20544677571385,
    "B": 69.47518717549964,
    "C": 70.20544677571385
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.28974409399088474,
    "p50_s_raw": 0.06987204699544236,
    "p95_s_adjusted": 10.52538783007183,
    "p95_s_raw": 5.187693915035915
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 91.36974107629146,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 82.4425565622077
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 91.36974107629146
      },
      "calibration_gen": {
       "accuracy": 0.7524752475247525,
       "cc": 55.45797264538123,
       "cc_by_type": {
        "choice": 71.36797454931072,
        "noul": 36.00000000000001,
        "score": 59.00594338683297
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 55.45797264538123
      },
      "composite_gen": {
       "accuracy": 0.7920792079207921,
       "cc": 53.48759110244231,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 3.703703703703698,
        "score": 63.26313464427364
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 53.48759110244231
      },
      "fairness": {
       "accuracy": 0.9583333333333334,
       "cc": 88.80841269511244,
       "cc_by_type": {
        "choice": 100,
        "noul": 73.33333333333334,
        "score": 93.09190475200394
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 88.80841269511244
      },
      "jaggedness": {
       "accuracy": 0.8222222222222222,
       "cc": 58.727647366249755,
       "cc_by_type": {
        "choice": 91.33782483156881,
        "noul": 29.729729729729737,
        "score": 55.11538753745072
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 58.727647366249755
      },
      "llm_router": {
       "accuracy": 0.9169329073482428,
       "cc": 79.845307649839,
       "cc_by_type": {
        "choice": 94.45276041208066,
        "noul": 87.62886597938144,
        "score": 57.45429655805487
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 79.845307649839
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 56.54793730611436,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 25,
        "score": 69.3860799595802
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 56.54793730611436
      },
      "multilingual": {
       "accuracy": 0.8581818181818182,
       "cc": 70.68658003787642,
       "cc_by_type": {
        "choice": 91.19660906423215,
        "noul": 44.44444444444444,
        "score": 76.41868660495271
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 70.68658003787642
      },
      "negation_pair_gen": {
       "accuracy": 0.7209302325581395,
       "cc": 38.055841143078034,
       "cc_by_type": {
        "choice": 100,
        "noul": -26.66666666666667,
        "score": 40.834190095900766
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 38.055841143078034
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 70.62365287202225,
       "cc_by_type": {
        "choice": 90.99099099099098,
        "noul": 80,
        "score": 40.87996762507578
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 70.62365287202225
      },
      "usecase": {
       "accuracy": 0.8590909090909091,
       "cc": 71.9252923706649,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 62.5,
        "score": 63.07979868062215
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 71.9252923706649
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8125,
       "cc": 61.094538064721405,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 50,
        "score": 52.33123324178327
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 61.094538064721405
      },
      "cs": {
       "accuracy": 0.8571428571428571,
       "cc": 89.47368421052632,
       "cc_by_type": {
        "choice": 78.94736842105263,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 89.47368421052632
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 73.25755504785428,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 86.43933181022953
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 73.25755504785428
      },
      "de": {
       "accuracy": 0.875,
       "cc": 84.14337867842146,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 76.32394134499887
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 84.14337867842146
      },
      "el": {
       "accuracy": 0.8,
       "cc": 72.15987140852917,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 33.33333333333333,
        "score": 99.90605742856702
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 72.15987140852917
      },
      "en": {
       "accuracy": 0.8410078192875761,
       "cc": 65.6279430221757,
       "cc_by_type": {
        "choice": 89.23615772323987,
        "noul": 47.482014388489205,
        "score": 60.16565695479803
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 65.6279430221757
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 71.39927626489069,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 23.98575701653176
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.39927626489069
      },
      "fi": {
       "accuracy": 1,
       "cc": 99.45381474863512,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.36144424590536
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 99.45381474863512
      },
      "fr": {
       "accuracy": 1,
       "cc": 98.79708726840215,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.39126180520648
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 98.79708726840215
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 68.26841937256562,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 25,
        "score": 79.80525811769688
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 68.26841937256562
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 74.74473760523466,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 24.23421281570397
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 74.74473760523466
      },
      "ja": {
       "accuracy": 0.8,
       "cc": 54.038814370488296,
       "cc_by_type": {
        "choice": 100,
        "noul": 39.99999999999999,
        "score": 22.11644311146491
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 54.038814370488296
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 14.500731144311763,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 43.502193432935286
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 14.500731144311763
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.20107791961944,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 98.26990042552501
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.20107791961944
      },
      "nl": {
       "accuracy": 0.8,
       "cc": 50.22874987381278,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 84.01958295477169
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 50.22874987381278
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 61.29679898233318,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 19.999999999999996,
        "score": 71.50783241039356
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 61.29679898233318
      },
      "pt": {
       "accuracy": 0.8928571428571429,
       "cc": 86.54314095373468,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100,
        "score": 80.84332975572379
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 86.54314095373468
      },
      "sv": {
       "accuracy": 0.9090909090909091,
       "cc": 87.36909613502253,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 100,
        "score": 87.63920329868462
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 87.36909613502253
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 75.97905009330604,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 77.93715027991811
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 75.97905009330604
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 87.46841338981227,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 94.66330468556585
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 87.46841338981227
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 48.76376745279072,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 12.957969025038828
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 48.76376745279072
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.9186666666666666,
       "cc": 89.49463250349223,
       "cc_by_type": {
        "choice": 89.49463250349223
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 89.49463250349223
      },
      "noul": {
       "accuracy": 0.744,
       "cc": 48.8,
       "cc_by_type": {
        "noul": 48.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 48.8
      },
      "score": {
       "accuracy": 0.7973333333333333,
       "cc": 62.42466072114584,
       "cc_by_type": {
        "score": 62.42466072114584
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 62.42466072114584
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6818181818181818,
       "cc": 34.51865055496734,
       "cc_by_type": {
        "choice": 84,
        "noul": 0,
        "score": 19.555951664902004
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 34.51865055496734
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 80.62020123707451,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 64.08282593344576
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 80.62020123707451
      },
      "customer_support": {
       "accuracy": 0.8571428571428571,
       "cc": 67.14209875854465,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50,
        "score": 67.68645887725995
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 67.14209875854465
      },
      "demand_forecasting": {
       "accuracy": 0.8235294117647058,
       "cc": 62.88499771485198,
       "cc_by_type": {
        "choice": 76.60818713450293,
        "noul": 33.33333333333333,
        "score": 78.71347267671969
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 62.88499771485198
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.9411764705882353,
       "cc": 89.9509753012271,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 69.8529259036813
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 89.9509753012271
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 29.476842130779346,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 66.66666666666667,
        "score": -65.65752392212738
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 29.476842130779346
      },
      "financial_crime": {
       "accuracy": 0.8333333333333334,
       "cc": 69.04160611650587,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 42.85714285714286,
        "score": 64.26767549237475
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 69.04160611650587
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 59.408208354864904,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 0,
        "score": 88.67758673706858
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 59.408208354864904
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 65.352863197908,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 60.00000000000001,
        "score": 77.72525626039064
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 65.352863197908
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 84.55664975223662,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 100,
        "score": 66.51791499546789
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 84.55664975223662
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 63.0110398677102,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 11.255341825352849
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 63.0110398677102
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 48.74742459938019,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 100,
        "score": 75.27453186265669
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 48.74742459938019
      },
      "llm_guardrails": {
       "accuracy": 0.8888888888888888,
       "cc": 67.9651992139091,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 83.89559764172728
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 67.9651992139091
      },
      "model_routing": {
       "accuracy": 0.8894009216589862,
       "cc": 75.02535719232871,
       "cc_by_type": {
        "choice": 92.98701298701299,
        "noul": 87.87878787878789,
        "score": 44.21027071118523
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.02535719232871
      },
      "moderation_trust_safety": {
       "accuracy": 0.8888888888888888,
       "cc": 31.98964337733368,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": -100,
        "score": 95.96893013200102
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 31.98964337733368
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 54.772780391389254,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 30.985007840834434
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.772780391389254
      },
      "risk_assessment": {
       "accuracy": 0.8333333333333334,
       "cc": 50.878022896636004,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 52.634068689908005
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 50.878022896636004
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 43.5106719476208,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": -33.333333333333336,
        "score": 78.35810279938414
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 43.5106719476208
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.85458175009258,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.56374525027778
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.85458175009258
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.30671773822844,
      91.70031153434485
     ],
     "I": [
      61.6000039357046,
      71.43891205182712
     ],
     "capability": [
      75.29188998324385,
      80.61305668502236
     ],
     "composite": null,
     "seed": 72
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.90819924197592,
     "capability": 78.8040995813651,
     "intelligence": 66.69999992075427
    },
    "per_type": {
     "choice": {
      "accuracy": 0.9186666666666666,
      "calibration": 91.8112970311977,
      "cc": 89.49463250349223,
      "cc_by_type": {
       "choice": 89.49463250349223
      },
      "cc_tiered": 91.43468725529016,
      "cc_tiered_open": 94.7183214036337,
      "cc_tiered_sealed": 88.15105310694662,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 89.49463250349223
     },
     "noul": {
      "accuracy": 0.744,
      "calibration": 89.64449693220469,
      "cc": 48.8,
      "cc_by_type": {
       "noul": 48.8
      },
      "cc_tiered": 47.150637613782905,
      "cc_tiered_open": 47.46790890269151,
      "cc_tiered_sealed": 46.833366324874305,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 48.8
     },
     "score": {
      "accuracy": 0.7973333333333333,
      "calibration": 91.2688037625254,
      "cc": 62.42466072114584,
      "cc_by_type": {
       "score": 62.42466072114584
      },
      "cc_tiered": 61.514674893189735,
      "cc_tiered_open": 61.328583803232945,
      "cc_tiered_sealed": 61.700765983146525,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.42466072114584
     }
    },
    "raw_on_A": {
     "calibration": 89.10291663559049,
     "capability": 78.82003448345057,
     "intelligence": 68.53715233131064
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01866666666666672,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.026666666666666616,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 65.25210488229811,
    "Emphasis on Accuracy 60:20:20": 65.82365355702983,
    "Emphasis on Cost 20:20:60": 61.47733677095195,
    "Emphasis on Speed 20:60:20": 68.88350125692479,
    "Intelligence only": 66.69999992075427
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "StartLux Labs",
   "axes": {
    "calibration": 91.92111224732803,
    "cost": 57.87760156376582,
    "intelligence": 73.19432958384127,
    "speed": 90.32360333729886
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.040930804934676546,
      "ece_score": 91.8138390130647,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 91.8138390130647,
      "score_before_time_limit": 91.8138390130647,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.17962466487935658,
      "brier": 0.04448579796465505,
      "ece": 0.04709348102978012,
      "n": 373,
      "score": 90.58130379404398,
      "score_before_time_limit": 90.58130379404398,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9596774193548387,
      "interval90_width": 0.20206093189964155,
      "n": 372,
      "nrps": 0.04615661957496166,
      "score": 93.3681939348754,
      "score_before_time_limit": 93.3681939348754,
      "time_limit_failed_share": 0,
      "top_ece": 0.04323975086376515
     }
    },
    "score": 91.92111224732803
   },
   "capability": 82.55772091558464,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11849.1171499 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=base-arch). The row's own scored cost basis: 737,990 input + 0 output tokens over 1,485 decisions = 496.962962963 tokens/decision. GPU-time cost USD 0.0253559535713225/1,000; old list-price cost USD 0.208724444444444/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0253559535713225/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.20663720000000002
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025355953571322483
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "StartLux-Decision v1.1 27B Instant (StartLux Labs, Qwen3.8-27B fine-tune)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H200 141 GB (sm_90, driver 580.173.02), Finland (EU). The author's own server on vLLM 0.19.1, offline, loopback, one request at a time.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 77.73279465771967,
    "I_sealed": 68.65586450996287,
    "base": 73.19432958384127,
    "excess": -0.7625966225989131,
    "gap": 9.076930147756798,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 87.38989648625872,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 88.67568417741427,
       "judge": 88.13447593935399,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 72.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 60.00000000000001,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 72.86024732126884,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.15270644756282,
       "hard": 80.70289974380364,
       "judge": 72.2051509514639,
       "standard": 58.01135746775971
      }
     },
     "sealed|choice": {
      "cc": 83.43695541516112,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 74.49982652943217,
       "judge": 87.65915768854063,
       "standard": 90.50833157561695
      }
     },
     "sealed|noul": {
      "cc": 57.713563842230855,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 72.41379310344827,
       "hard": 49.09090909090909,
       "judge": 60.00000000000001,
       "standard": 64.17910447761194
      }
     },
     "sealed|score": {
      "cc": 64.81707427249664,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 87.88715519480432,
       "hard": 62.862170937119835,
       "judge": 62.78531836211132,
       "standard": 60.23947434767438
      }
     }
    }
   },
   "jevbench_score": 75.62823658595717,
   "key": "startlux-decision-v1.1-27b-instant",
   "last_measured_on": "2026-10-16",
   "licence": "Weights CC BY-NC 4.0 (non-commercial: commercial use needs a separate licence from StartLux Labs; listing a measured score is evaluation use); inference package Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.25",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "startlux-models/StartLux-Decision-v1.1-27B-Instant@32e7d35d74db615c4a059181efbf21c5421f3561",
   "noul_decisive": {
    "acc_among_decisive": 0.9934640522875817,
    "decisive": 306,
    "decisive_rate": 0.8203753351206434,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.92111224732803,
     "cost": 30.41280058663709,
     "intelligence": 73.19432958384127,
     "speed": 90.62312905261143
    },
    "board_release": "v1.6.43",
    "capability": 82.55772091558464,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      21.245034378926533,
      21.810261320192286
     ],
     "B": [
      21.973213364261966,
      22.86842408467527
     ],
     "C": [
      21.245034378926533,
      21.810261320192286
     ]
    },
    "jevbench_score": 21.618048863688184,
    "measured_in": "v1.6.25",
    "method": "v1.6",
    "open_board_rank": 85,
    "rank": 118,
    "ranks": {
     "A": 118,
     "B": 114,
     "C": 101,
     "capability": 14
    },
    "scores": {
     "A": 21.618048863688184,
     "B": 22.526778205217695,
     "C": 21.618048863688184
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2602560680010356,
     "p50_s_raw": 0.0551280340005178,
     "p95_s_adjusted": 0.33287887162878177,
     "p95_s_raw": 0.09143943581439089
    },
    "views": {
     "Balanced 33:33:33": 19.27695949442568,
     "Emphasis on Accuracy 60:20:20": 21.788279630595614,
     "Emphasis on Cost 20:20:60": 14.99823406009409,
     "Emphasis on Speed 20:60:20": 23.225862696351562,
     "Intelligence only": 27.08009959487104
    }
   },
   "rank": 14,
   "ranked": true,
   "ranks": {
    "A": 14,
    "B": 11,
    "C": 13,
    "capability": 16
   },
   "repo": "https://huggingface.co/startlux-models/StartLux-Decision-v1.1-27B-Instant",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 75.62823658595717,
    "B": 75.12859131154335,
    "C": 75.62823658595717
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.26417634401004764,
    "p50_s_raw": 0.05708817200502381,
    "p95_s_adjusted": 0.3513546783942729,
    "p95_s_raw": 0.10067733919713646
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 94.60413033269491,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 92.14572433141801
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 94.60413033269491
      },
      "calibration_gen": {
       "accuracy": 0.693069306930693,
       "cc": 54.896811745749496,
       "cc_by_type": {
        "choice": 55.461293743372224,
        "noul": 43.99999999999999,
        "score": 65.22914149387627
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 54.896811745749496
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 84.58165491823706,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 85.18518518518519,
        "score": 78.3158771305016
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 84.58165491823706
      },
      "fairness": {
       "accuracy": 0.9166666666666666,
       "cc": 84.39784298036983,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 73.33333333333334,
        "score": 98.90781465539521
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.39784298036983
      },
      "jaggedness": {
       "accuracy": 0.6777777777777778,
       "cc": 49.005904102153835,
       "cc_by_type": {
        "choice": 68.23869104908566,
        "noul": 18.918918918918926,
        "score": 59.86010233845691
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 49.005904102153835
      },
      "llm_router": {
       "accuracy": 0.9456869009584664,
       "cc": 84.73240889274621,
       "cc_by_type": {
        "choice": 97.62261160517741,
        "noul": 95.8762886597938,
        "score": 60.698326413267424
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 84.73240889274621
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 58.29924872452923,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 50,
        "score": 62.01114823544332
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 58.29924872452923
      },
      "multilingual": {
       "accuracy": 0.8254545454545454,
       "cc": 69.69226325427245,
       "cc_by_type": {
        "choice": 87.28399087055756,
        "noul": 50,
        "score": 71.79279889225978
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 69.69226325427245
      },
      "negation_pair_gen": {
       "accuracy": 0.872093023255814,
       "cc": 72.38166965157417,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 57.1450089547225
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 72.38166965157417
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.875,
       "cc": 80.83683405690114,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 57.525517185718435
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.83683405690114
      },
      "usecase": {
       "accuracy": 0.8227272727272728,
       "cc": 65.68990894056331,
       "cc_by_type": {
        "choice": 87.74509803921569,
        "noul": 37.5,
        "score": 71.82462878247424
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 65.68990894056331
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 60.96108872191072,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100,
        "score": 20.978504260970265
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 60.96108872191072
      },
      "cs": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 100
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 61.30656762125532,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 50.58636953043263
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 61.30656762125532
      },
      "de": {
       "accuracy": 0.875,
       "cc": 86.47138592260931,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 83.30796307756245
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 86.47138592260931
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.71241850166923,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.99014191096676
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.71241850166923
      },
      "en": {
       "accuracy": 0.8331885317115552,
       "cc": 71.69241929432023,
       "cc_by_type": {
        "choice": 84.18374196067901,
        "noul": 63.309352517985616,
        "score": 67.58416340429606
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 71.69241929432023
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 77.55106151830554,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 32.653184554916635
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 77.55106151830554
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.7312947809802,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.86055100960733
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.7312947809802
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 78.71872583341194,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 100,
        "score": 99.68558926494168
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 78.71872583341194
      },
      "hi": {
       "accuracy": 0.9047619047619048,
       "cc": 82.14587496948548,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 75,
        "score": 88.19740144476927
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 82.14587496948548
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 1,
       "cc": 77.6224564529621,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 32.867369358886286
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 77.6224564529621
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 50.59151729695487,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 31.774551890864622
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 50.59151729695487
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 12.928731135718815,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 38.78619340715645
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 12.928731135718815
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.62243022913351,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.53395735406725
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.62243022913351
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 29.967630712302093,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 23.236225470239614
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 29.967630712302093
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 67.74193548387098,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 67.74193548387098
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 75.83899608734215,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 75.13442372542049
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 75.83899608734215
      },
      "pt": {
       "accuracy": 0.8928571428571429,
       "cc": 85.66738033499985,
       "cc_by_type": {
        "choice": 89.39304655274012,
        "noul": 100,
        "score": 67.6090944522594
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 85.66738033499985
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 61.56852305958395,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 50.23748407236888
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 61.56852305958395
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 56.86499088250887,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 70.5949726475266
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 56.86499088250887
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 78.31956327674492,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 99.47481886249281
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 78.31956327674492
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 57.8305341335463,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 40.158269067305575
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 57.8305341335463
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8813333333333333,
       "cc": 84.67249660345587,
       "cc_by_type": {
        "choice": 84.67249660345587
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.67249660345587
      },
      "noul": {
       "accuracy": 0.8106666666666666,
       "cc": 62.133333333333326,
       "cc_by_type": {
        "noul": 62.133333333333326
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 62.133333333333326
      },
      "score": {
       "accuracy": 0.7573333333333333,
       "cc": 67.32764882650292,
       "cc_by_type": {
        "score": 67.32764882650292
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 67.32764882650292
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 29.881576050153324,
       "cc_by_type": {
        "choice": 68,
        "noul": -25,
        "score": 46.64472815045997
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 29.881576050153324
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 87.04724612514936,
       "cc_by_type": {
        "choice": 81.36645962732919,
        "noul": 100,
        "score": 79.77527874811891
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 87.04724612514936
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 56.37200356001129,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 67.89649848491193
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 56.37200356001129
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 1.3223643717638087,
       "cc_by_type": {
        "choice": 64.91228070175438,
        "noul": -33.333333333333336,
        "score": -27.61185425312962
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 1.3223643717638087
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 46.99599177928393,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 88.81406229437354
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 46.99599177928393
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 45.49259456532935,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": 15.723066714855982
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 45.49259456532935
      },
      "financial_crime": {
       "accuracy": 0.8333333333333334,
       "cc": 62.43686377152064,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 14.28571428571428,
        "score": 73.02487702884764
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 62.43686377152064
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 48.71896369085224,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 0,
        "score": 67.06281441750444
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 48.71896369085224
      },
      "insurance_claims": {
       "accuracy": 0.8,
       "cc": 63.77044581469289,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 71.31133744407869
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 63.77044581469289
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 77.83384735706747,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 66.83487540453574
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 77.83384735706747
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 65.3802683789926,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 52.21315655816645
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 65.3802683789926
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 41.376624819304425,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 55.31267015683801
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 41.376624819304425
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 63.70402277234668,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 71.11206831704003
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 63.70402277234668
      },
      "model_routing": {
       "accuracy": 0.9124423963133641,
       "cc": 79.45253805021827,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 93.93939393939394,
        "score": 49.09354488658552
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.45253805021827
      },
      "moderation_trust_safety": {
       "accuracy": 0.8888888888888888,
       "cc": 59.55706008248318,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 0,
        "score": 90.18749502672017
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 59.55706008248318
      },
      "recruiting": {
       "accuracy": 0.8888888888888888,
       "cc": 88.75474237649098,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 66.26422712947296
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 88.75474237649098
      },
      "risk_assessment": {
       "accuracy": 0.7222222222222222,
       "cc": 45.51020954722372,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": 51.30895376482386
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 45.51020954722372
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 50.702961440976175,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": 0,
        "score": 66.60163794611695
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 50.702961440976175
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.97000103064437,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.91000309193309
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.97000103064437
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      88.15955420517918,
      92.63070738800778
     ],
     "I": [
      68.6761427354401,
      76.80063493292705
     ],
     "capability": [
      78.99248244635237,
      84.23659103542126
     ],
     "composite": null,
     "seed": 73
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.92111224732803,
     "capability": 82.55772091558464,
     "intelligence": 73.19432958384127
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8813333333333333,
      "calibration": 91.8138390130647,
      "cc": 84.67249660345587,
      "cc_by_type": {
       "choice": 84.67249660345587
      },
      "cc_tiered": 85.41342595070992,
      "cc_tiered_open": 87.38989648625872,
      "cc_tiered_sealed": 83.43695541516112,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.67249660345587
     },
     "noul": {
      "accuracy": 0.8106666666666666,
      "calibration": 90.58130379404398,
      "cc": 62.133333333333326,
      "cc_by_type": {
       "noul": 62.133333333333326
      },
      "cc_tiered": 65.33090200393116,
      "cc_tiered_open": 72.94824016563147,
      "cc_tiered_sealed": 57.713563842230855,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 62.133333333333326
     },
     "score": {
      "accuracy": 0.7573333333333333,
      "calibration": 93.3681939348754,
      "cc": 67.32764882650292,
      "cc_by_type": {
       "score": 67.32764882650292
      },
      "cc_tiered": 68.83866079688275,
      "cc_tiered_open": 72.86024732126884,
      "cc_tiered_sealed": 64.81707427249664,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 67.32764882650292
     }
    },
    "raw_on_A": {
     "calibration": 88.14525609544745,
     "capability": 79.34618044792921,
     "intelligence": 70.54710480041096
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 71.40918079171863,
    "Emphasis on Accuracy 60:20:20": 72.11268817500502,
    "Emphasis on Cost 20:20:60": 65.30221017213528,
    "Emphasis on Speed 20:60:20": 77.93744934896922,
    "Intelligence only": 73.19432958384127
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "Strike (strike-ai.eu)",
   "axes": {
    "calibration": 91.35849542802363,
    "cost": 54.05958090666912,
    "intelligence": 45.347915471780475,
    "speed": 83.29564724453803
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03179085714285703,
      "ece_score": 93.6418285714286,
      "mean_tvd": null,
      "n": 746,
      "n_ece_hard": 350,
      "n_tvd": 0,
      "score": 93.6418285714286,
      "score_before_time_limit": 93.6418285714286,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2887700534759359,
      "brier": 0.0961551485294117,
      "ece": 0.05632165775401073,
      "n": 374,
      "score": 88.73566844919786,
      "score_before_time_limit": 88.73566844919786,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.952,
      "interval90_width": 0.3739999999999997,
      "n": 375,
      "nrps": 0.08303541473111116,
      "score": 91.69798926344444,
      "score_before_time_limit": 91.69798926344444,
      "time_limit_failed_share": 0,
      "top_ece": 0.04150240000000007
     }
    },
    "score": 91.35849542802363
   },
   "capability": 68.35320544990205,
   "composite_ci95": {
    "A": [
     36.46557694185339,
     64.90036015309953
    ],
    "B": [
     33.09562694594094,
     61.38440159220277
    ],
    "C": [
     25.32331732073152,
     46.258659710436746
    ]
   },
   "cost": {
    "basis": "ESTIMATE (*est.), hosted API without a counted list price: the vendor's stated base Qwen/Qwen3.5-4B at its market reference deepinfra:Qwen/Qwen3.5-4B, USD 0.03 per million input tokens and USD 0.15 per million output tokens (frozen pricing module sha256 645fd82eac728d597048bec0995ecd3ac896566e269fb6c9a1fdb84c171a9ac4), x the vendor-reported usage: 1,687,022 input tokens and 0 output tokens on the 1,489 common items (output 0 as reported: no text is generated). COST FIGURES: USD 0.033990 per 1,000 decisions, 0.53x the official cost cap USD 0.064595. Cost on the 1,489 items every row on this draw answered (same-items rule); all-items cost USD 0.033853 per 1,000 decisions. Estimated model tariff, not charged usage or a host quote; the vendor reported no charge for these requests.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.033852520000000004
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.03398969778374748
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT of Strike-1, a hosted decision API (EU, OVH Gravelines). The vendor describes it as a fine-tune of Qwen/Qwen3.5-4B; the weights are not published, so the base is the vendor's statement. The vendor describes the service as beta and states that its probabilities are uncalibrated during beta; Calibration is measured on the probabilities as returned. ITEMS: all 1,500 items of the draw were sent; 5 items over the provider's input limit were refused and are scored as misses (kept in the denominator). TOKENS: the API reports 0 output tokens on every answer (it returns a probability distribution and generates no text). DRAW: measured on the 27 Oct 2026 regular draw (fresh sealed items; not yet equated to the reference scale, so its numbers are shown as published, offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own draw. LATENCY: p50 0.475 s, p95 0.987 s, maximum 2.225 s.",
   "display": "Strike-1",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-28",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.16",
    "results_sha256": "7b6ebfdfcc33e45e9eed6f92825d47ae90dd59c3ee750d0852a677f5fcfefe87",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted API in the EU (OVH Gravelines, France), native /v1/systemone; latency measured from our server in Germany. 1,495 of 1,500 requests answered (HTTP 200, served model strike-1 on every answer); 5 were refused by the server because the input exceeded its limit (8,192 tokens or 32,000 characters) and are scored as misses; no decision took longer than 10 s.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.88492172806097,
    "I_sealed": 48.81090921549998,
    "base": 45.347915471780475,
    "excess": -9.49937512987726,
    "gap": -6.925987487439016,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.20321866307117,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 69.80182447310474,
       "judge": 76.26895187870798,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 23.126293995859214,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.85714285714286,
       "hard": 30.434782608695656,
       "judge": 0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 32.32525252525252,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.82030303030303,
       "hard": 25.813981481481473,
       "judge": 49.36000000000001,
       "standard": 7.548148148148137
      }
     },
     "sealed|choice": {
      "cc": 70.002173399817,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 84.27775958074025,
       "hard": 56.26452567082189,
       "judge": 75.15099223468506,
       "standard": 82.61644751504349
      }
     },
     "sealed|noul": {
      "cc": 31.229813340635413,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 58.62068965517242,
       "hard": -7.317073170731703,
       "judge": 55.55555555555556,
       "standard": 58.13953488372092
      }
     },
     "sealed|score": {
      "cc": 45.20074090604755,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 68.64672727272729,
       "hard": 22.785791505791497,
       "judge": 62.09886363636363,
       "standard": 52.96046242774564
      }
     }
    }
   },
   "jevbench_score": 51.81470102195167,
   "key": "strike-1",
   "last_measured_on": "2026-10-28",
   "listing": "unranked",
   "measured_in": "v1.7.16",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": {
    "model": "strike-1",
    "provider": "Strike (strike-ai.eu), hosted API",
    "region": "EU, OVH Gravelines (ovh-gra)",
    "response_model": "strike-1",
    "version": "v20261023 (vendor revision)"
   },
   "new_in": "v1.7.16",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale; stronger anchor runs for that draw are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.9661654135338346,
    "decisive": 266,
    "decisive_rate": 0.7112299465240641,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "provider": "strike",
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://strike-ai.eu",
   "scores": {
    "A": 51.81470102195167,
    "B": 48.07396890608295,
    "C": 35.982431265244216
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.47458370300591923,
    "p50_s_raw": 0.47458370300591923,
    "p95_s_adjusted": 0.9865576189680723,
    "p95_s_raw": 0.9865576189680723
   },
   "status": {
    "answered_ok": 1495,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.75,
       "cc": 66.66666666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 66.66666666666666
      },
      "calibration_gen": {
       "accuracy": 0.6428571428571429,
       "cc": 29.644769119769126,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 0,
        "score": 14.908333333333346
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 29.644769119769126
      },
      "composite_gen": {
       "accuracy": 0.631578947368421,
       "cc": 33.779444444444444,
       "cc_by_type": {
        "choice": 100,
        "noul": -19.999999999999996,
        "score": 21.33833333333334
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 33.779444444444444
      },
      "fairness": {
       "accuracy": 1,
       "cc": 97.86888888888889,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 93.60666666666665
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 97.86888888888889
      },
      "jaggedness": {
       "accuracy": 0.4090909090909091,
       "cc": -16.51419753086419,
       "cc_by_type": {
        "choice": 62.96296296296297,
        "noul": -100,
        "score": -12.505555555555548
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.7888888888888889,
       "cc": 65.93774883935967,
       "cc_by_type": {
        "choice": 76.20446533490012,
        "noul": 73.91304347826086,
        "score": 47.69573770491803
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 65.93774883935967
      },
      "long_state": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.7285559174809989,
       "cc": 53.48449091792713,
       "cc_by_type": {
        "choice": 67.66544513903861,
        "noul": 43.44262295081967,
        "score": 49.345404663923134
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 53.48449091792713
      },
      "negation_pair_gen": {
       "accuracy": 0.45454545454545453,
       "cc": 1.6666666666666643,
       "cc_by_type": {
        "choice": 86.66666666666667,
        "noul": -83.33333333333334
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 1.6666666666666643
      },
      "option_order": {
       "accuracy": 0.8518518518518519,
       "cc": 80.95238095238095,
       "cc_by_type": {
        "choice": 80.95238095238095
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 80.95238095238095
      },
      "perturb_pair": {
       "accuracy": 0.7857142857142857,
       "cc": 67.40452258433392,
       "cc_by_type": {
        "choice": 71.69811320754717,
        "noul": 60.00000000000001,
        "score": 70.51545454545456
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 67.40452258433392
      },
      "usecase": {
       "accuracy": 0.72,
       "cc": 48.04083227058061,
       "cc_by_type": {
        "choice": 67.90663749088255,
        "noul": 29.230769230769237,
        "score": 46.985090090090054
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 48.04083227058061
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.7619047619047619,
       "cc": 59.18359298057123,
       "cc_by_type": {
        "choice": 72.6027397260274,
        "noul": 33.33333333333333,
        "score": 71.61470588235295
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 59.18359298057123
      },
      "cs": {
       "accuracy": 0.6756756756756757,
       "cc": 41.90323826500297,
       "cc_by_type": {
        "choice": 59.89304812834225,
        "noul": 33.33333333333333,
        "score": 32.483333333333334
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 41.90323826500297
      },
      "da": {
       "accuracy": 0.6388888888888888,
       "cc": 27.90013739883305,
       "cc_by_type": {
        "choice": 55.27950310559007,
        "noul": -9.090909090909093,
        "score": 37.51181818181818
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 27.90013739883305
      },
      "de": {
       "accuracy": 0.7916666666666666,
       "cc": 58.08872056903829,
       "cc_by_type": {
        "choice": 81.47671726267045,
        "noul": 50,
        "score": 42.78944444444443
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 58.08872056903829
      },
      "el": {
       "accuracy": 0.5357142857142857,
       "cc": 23.959897435897428,
       "cc_by_type": {
        "choice": 42.307692307692314,
        "noul": 0,
        "score": 29.571999999999964
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 23.959897435897428
      },
      "en": {
       "accuracy": 0.7153153153153153,
       "cc": 45.83386173088808,
       "cc_by_type": {
        "choice": 71.20893812070283,
        "noul": 21.311475409836067,
        "score": 44.98117166212534
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 45.83386173088808
      },
      "es": {
       "accuracy": 0.8378378378378378,
       "cc": 70.32340758561288,
       "cc_by_type": {
        "choice": 85.56535685645551,
        "noul": 55.55555555555556,
        "score": 69.84931034482759
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 70.32340758561288
      },
      "fi": {
       "accuracy": 0.7692307692307693,
       "cc": 61.776706984853426,
       "cc_by_type": {
        "choice": 65.71428571428571,
        "noul": 68.42105263157893,
        "score": 51.19478260869565
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 61.776706984853426
      },
      "fr": {
       "accuracy": 0.65625,
       "cc": 57.48932522503952,
       "cc_by_type": {
        "choice": 48.979591836734706,
        "noul": 77.77777777777777,
        "score": 45.710606060606075
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 57.48932522503952
      },
      "hi": {
       "accuracy": 0.7567567567567568,
       "cc": 57.326938156266216,
       "cc_by_type": {
        "choice": 75.20661157024793,
        "noul": 33.33333333333333,
        "score": 63.4408695652174
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 57.326938156266216
      },
      "id": {
       "accuracy": 0.8292682926829268,
       "cc": 73.06084485407065,
       "cc_by_type": {
        "choice": 81.5668202764977,
        "noul": 66.66666666666667,
        "score": 70.9490476190476
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 73.06084485407065
      },
      "it": {
       "accuracy": 0.8222222222222222,
       "cc": 66.14112121660258,
       "cc_by_type": {
        "choice": 92.01419698314109,
        "noul": 66.66666666666667,
        "score": 39.74249999999998
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 66.14112121660258
      },
      "ja": {
       "accuracy": 0.7727272727272727,
       "cc": 55.56449442612655,
       "cc_by_type": {
        "choice": 76.68393782383419,
        "noul": 45.45454545454546,
        "score": 44.555000000000014
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 55.56449442612655
      },
      "ko": {
       "accuracy": 0.627906976744186,
       "cc": 33.59368286445013,
       "cc_by_type": {
        "choice": 55.88235294117647,
        "noul": 0,
        "score": 44.89869565217391
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 33.59368286445013
      },
      "mixed": {
       "accuracy": 0.828125,
       "cc": 67.22912661266126,
       "cc_by_type": {
        "choice": 83.79837983798379,
        "noul": 70,
        "score": 47.888999999999996
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 67.22912661266126
      },
      "nl": {
       "accuracy": 0.75,
       "cc": 51.93888050314465,
       "cc_by_type": {
        "choice": 64.62264150943396,
        "noul": 60.00000000000001,
        "score": 31.19399999999999
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 51.93888050314465
      },
      "no": {
       "accuracy": 0.7254901960784313,
       "cc": 41.659468047629105,
       "cc_by_type": {
        "choice": 63.85542168674698,
        "noul": 46.66666666666666,
        "score": 14.456315789473685
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 41.659468047629105
      },
      "pl": {
       "accuracy": 0.631578947368421,
       "cc": 28.26389143603964,
       "cc_by_type": {
        "choice": 61.681745609366686,
        "noul": 5.882352941176472,
        "score": 17.227575757575753
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 28.26389143603964
      },
      "pt": {
       "accuracy": 0.7446808510638298,
       "cc": 55.68424682060114,
       "cc_by_type": {
        "choice": 76.03993344425956,
        "noul": 33.33333333333333,
        "score": 57.67947368421053
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 55.68424682060114
      },
      "sv": {
       "accuracy": 0.6521739130434783,
       "cc": 43.33054580118826,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 16.666666666666675,
        "score": 46.8445238095238
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 43.33054580118826
      },
      "tr": {
       "accuracy": 0.717391304347826,
       "cc": 67.73166566366822,
       "cc_by_type": {
        "choice": 58.950969213226905,
        "noul": 80,
        "score": 64.24402777777777
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 67.73166566366822
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 54.37616764514024,
       "cc_by_type": {
        "choice": 50.68493150684932,
        "noul": 71.42857142857142,
        "score": 41.015
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 54.37616764514024
      },
      "zh": {
       "accuracy": 0.7073170731707317,
       "cc": 54.02850578512397,
       "cc_by_type": {
        "choice": 62.80991735537191,
        "noul": 50,
        "score": 49.2756
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 54.02850578512397
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7746666666666666,
       "cc": 69.2637237170484,
       "cc_by_type": {
        "choice": 69.2637237170484
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 69.2637237170484
      },
      "noul": {
       "accuracy": 0.6853333333333333,
       "cc": 37.06666666666667,
       "cc_by_type": {
        "noul": 37.06666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 37.06666666666667
      },
      "score": {
       "accuracy": 0.6613333333333333,
       "cc": 47.43238738738741,
       "cc_by_type": {
        "score": 47.43238738738741
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 47.43238738738741
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6041666666666666,
       "cc": 30.930722307529383,
       "cc_by_type": {
        "choice": 46.627131208302444,
        "noul": -14.28571428571429,
        "score": 60.45075
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 30.930722307529383
      },
      "code_linting": {
       "accuracy": 0.6521739130434783,
       "cc": 36.28932771815229,
       "cc_by_type": {
        "choice": 63.25780771586037,
        "noul": 33.33333333333333,
        "score": 12.276842105263174
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 36.28932771815229
      },
      "customer_support": {
       "accuracy": 0.803030303030303,
       "cc": 55.900552534472716,
       "cc_by_type": {
        "choice": 81.54981549815498,
        "noul": 19.999999999999996,
        "score": 66.15184210526316
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 55.900552534472716
      },
      "demand_forecasting": {
       "accuracy": 0.5,
       "cc": 8.572350197467713,
       "cc_by_type": {
        "choice": 40.864291129643675,
        "noul": -23.076923076923073,
        "score": 7.929682539682537
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 8.572350197467713
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6935483870967742,
       "cc": 41.51683518130346,
       "cc_by_type": {
        "choice": 74.66572836030966,
        "noul": 5.882352941176472,
        "score": 44.002424242424254
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 41.51683518130346
      },
      "feature_extraction": {
       "accuracy": 0.7222222222222222,
       "cc": 56.13585690356544,
       "cc_by_type": {
        "choice": 58.54271356783919,
        "noul": 57.14285714285714,
        "score": 52.72199999999998
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 56.13585690356544
      },
      "financial_crime": {
       "accuracy": 0.7142857142857143,
       "cc": 30.658495481127073,
       "cc_by_type": {
        "choice": 74.48165869218504,
        "noul": -9.090909090909093,
        "score": 26.584736842105272
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 30.658495481127073
      },
      "gaming": {
       "accuracy": 0.6206896551724138,
       "cc": 30.394125645565893,
       "cc_by_type": {
        "choice": 61.75298804780878,
        "noul": -11.111111111111116,
        "score": 40.540500000000016
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 30.394125645565893
      },
      "insurance_claims": {
       "accuracy": 0.6129032258064516,
       "cc": 15.930405738241845,
       "cc_by_type": {
        "choice": 55.45657015590201,
        "noul": -5.882352941176472,
        "score": -1.7830000000000013
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 15.930405738241845
      },
      "knowledge_graphs": {
       "accuracy": 0.6865671641791045,
       "cc": 32.07832429938877,
       "cc_by_type": {
        "choice": 68.51574212893554,
        "noul": 7.692307692307687,
        "score": 20.026923076923065
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 32.07832429938877
      },
      "lead_generation": {
       "accuracy": 0.8133333333333334,
       "cc": 64.68170125596077,
       "cc_by_type": {
        "choice": 70.38081805359661,
        "noul": 65.71428571428572,
        "score": 57.94999999999999
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 64.68170125596077
      },
      "legal_compliance": {
       "accuracy": 0.7678571428571429,
       "cc": 60.50837081808195,
       "cc_by_type": {
        "choice": 87.96389167502507,
        "noul": 57.14285714285714,
        "score": 36.41836363636365
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 60.50837081808195
      },
      "llm_guardrails": {
       "accuracy": 0.8873239436619719,
       "cc": 79.39950691506954,
       "cc_by_type": {
        "choice": 91.85336048879836,
        "noul": 76.92307692307692,
        "score": 69.42208333333333
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 79.39950691506954
      },
      "model_routing": {
       "accuracy": 0.7837837837837838,
       "cc": 65.73669035908296,
       "cc_by_type": {
        "choice": 73.51566557036526,
        "noul": 70.58823529411764,
        "score": 53.10617021276598
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 65.73669035908296
      },
      "moderation_trust_safety": {
       "accuracy": 0.7796610169491526,
       "cc": 65.82767634558377,
       "cc_by_type": {
        "choice": 67.32026143790848,
        "noul": 64.70588235294117,
        "score": 65.45688524590163
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 65.82767634558377
      },
      "other": {
       "accuracy": 0.75,
       "cc": 64.65504254532001,
       "cc_by_type": {
        "choice": 53.38512763596004,
        "noul": 100,
        "score": 40.580000000000005
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 64.65504254532001
      },
      "recruiting": {
       "accuracy": 0.8461538461538461,
       "cc": 64.00828906512858,
       "cc_by_type": {
        "choice": 82.69646719538572,
        "noul": 68,
        "score": 41.32840000000001
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 64.00828906512858
      },
      "risk_assessment": {
       "accuracy": 0.7083333333333334,
       "cc": 47.68603315094947,
       "cc_by_type": {
        "choice": 56.06694560669455,
        "noul": 53.846153846153854,
        "score": 33.14499999999999
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 47.68603315094947
      },
      "scientific_discovery": {
       "accuracy": 0.5714285714285714,
       "cc": 36.10606112247217,
       "cc_by_type": {
        "choice": 41.10429447852761,
        "noul": 19.999999999999996,
        "score": 47.21388888888889
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 36.10606112247217
      },
      "search_retrieval": {
       "accuracy": 0.8656716417910447,
       "cc": 75.9719246031746,
       "cc_by_type": {
        "choice": 100,
        "noul": 42.85714285714286,
        "score": 85.05863095238095
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 75.9719246031746
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.73594314027315,
      90.49488806544359
     ],
     "I": [
      39.22154104059264,
      50.731278880285
     ],
     "capability": [
      63.08563130078809,
      69.65241948645166
     ],
     "composite": {
      "A": [
       36.46557694185339,
       64.90036015309953
      ],
      "B": [
       33.09562694594094,
       61.38440159220277
      ],
      "C": [
       25.32331732073152,
       46.258659710436746
      ]
     },
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.35849542802363,
     "capability": 68.35320544990205,
     "intelligence": 45.347915471780475
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7746666666666666,
      "calibration": 93.6418285714286,
      "cc": 69.2637237170484,
      "cc_by_type": {
       "choice": 69.2637237170484
      },
      "cc_tiered": 70.10269603144408,
      "cc_tiered_open": 70.20321866307117,
      "cc_tiered_sealed": 70.002173399817,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.2637237170484
     },
     "noul": {
      "accuracy": 0.6853333333333333,
      "calibration": 88.73566844919786,
      "cc": 37.06666666666667,
      "cc_by_type": {
       "noul": 37.06666666666667
      },
      "cc_tiered": 27.178053668247315,
      "cc_tiered_open": 23.126293995859214,
      "cc_tiered_sealed": 31.229813340635413,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 37.06666666666667
     },
     "score": {
      "accuracy": 0.6613333333333333,
      "calibration": 91.69798926344444,
      "cc": 47.43238738738741,
      "cc_by_type": {
       "score": 47.43238738738741
      },
      "cc_tiered": 38.76299671565003,
      "cc_tiered_open": 32.32525252525252,
      "cc_tiered_sealed": 45.20074090604755,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 47.43238738738741
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.005333333333333301,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 46.954762335400005,
    "Emphasis on Accuracy 60:20:20": 42.55039282870394,
    "Emphasis on Cost 20:20:60": 45.927399829325495,
    "Emphasis on Speed 20:60:20": 53.7165330849937,
    "Intelligence only": 37.301987881299524
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "SubQ",
   "axes": {
    "calibration": 88.06513871774217,
    "cost": 60.67742028447466,
    "intelligence": 75.75710514449688,
    "speed": 84.90104208468355
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05760546714540509,
      "ece_score": 88.47890657091898,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 88.47890657091898,
      "score_before_time_limit": 88.47890657091898,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.11199999999999999,
      "brier": 0.07228155825255182,
      "ece": 0.07309512757394987,
      "n": 375,
      "score": 85.38097448521003,
      "score_before_time_limit": 85.38097448521003,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.944,
      "interval90_width": 0.16422222222222224,
      "n": 375,
      "nrps": 0.049061246713496494,
      "score": 90.33553509709749,
      "score_before_time_limit": 90.33553509709749,
      "time_limit_failed_share": 0,
      "top_ece": 0.07211402567227683
     }
    },
    "score": 88.06513871774217
   },
   "capability": 81.91112193111952,
   "composite_ci95": {
    "A": [
     74.09061953247686,
     76.6644175462098
    ],
    "B": [
     73.52994358661283,
     77.08716692169163
    ],
    "C": [
     74.09061953247686,
     76.6644175462098
    ]
   },
   "cost": {
    "basis": "Vendor's published standard launch tariff, USD 0.04 per million input tokens; output free (read 22 Oct 2026 at https://docs.subq.ai/using-the-api/models/ and https://subq.ai). Counted under the frozen I-1 launch-list-price rule. Capacity-limited public preview, first come first served, with a waitlist once capacity is reached; not openly bookable by everyone. Promotional signup credits are excluded. Own measured usage: 1,939,205 input / 0 output tokens over all 1,500 decisions. Common 1,485-item cost basis: 759,310 input / 0 output tokens, USD 0.020452795 per 1,000 decisions; all-item cost USD 0.051712133 per 1,000. The historical preview row keeps its frozen USD 0.05/M input tariff.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.051712133333333334
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020452794612794613
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SubQ Snap 1.1 (SubQ)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.subq.ai /v1/systemone, served model subq-snap-1.1, measured 22 Oct 2026 from the evaluator host, official per-item latency subset without API adjustment. Full S1200+P300 draw; single same-author second use, provider saw the sealed items in the 13 Oct preview run. Native probabilities, no generated tokens. Hosting implementation and deployed weights were not independently inspected. Inputs retained indefinitely by default; this run used already-retired items.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 81.19768798375172,
    "I_sealed": 70.31652230524202,
    "base": 75.75710514449688,
    "excess": 1.0416389081539918,
    "gap": 10.881165678509703,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 88.50761395257393,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 88.67568417741427,
       "judge": 88.13447593935399,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 83.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 70,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 72.0081553126909,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 63.478885212283465,
       "hard": 75.63135702400898,
       "judge": 80.3971244828602,
       "standard": 56.442933185004485
      }
     },
     "sealed|choice": {
      "cc": 81.88359847741832,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 79.70394356424192,
       "judge": 80.60724779627816,
       "standard": 83.86416367854883
      }
     },
     "sealed|noul": {
      "cc": 61.722683903847056,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 52.727272727272734,
       "judge": 63.07692307692307,
       "standard": 67.16417910447761
      }
     },
     "sealed|score": {
      "cc": 67.34328453446071,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 82.75405644034946,
       "hard": 69.4360268544149,
       "judge": 62.2790519956979,
       "standard": 63.048762749752164
      }
     }
    }
   },
   "jevbench_score": 75.73702535215112,
   "key": "subq-snap-1.1",
   "last_measured_on": "2026-10-22",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.43",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:https://subq.ai#subq-snap-1.1",
   "noul_decisive": {
    "acc_among_decisive": 0.9459459459459459,
    "decisive": 333,
    "decisive_rate": 0.888,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 4,
    "axes": {
     "calibration": 88.06513871774217,
     "cost": 60.67742028447466,
     "intelligence": 75.75710514449688,
     "speed": 84.94774043787051
    },
    "board_release": "v1.6.43",
    "capability": 81.91112193111952,
    "capability_api_board_rank": 4,
    "capability_open_board_rank": null,
    "capability_rank": 7,
    "composite_ci95": {
     "A": [
      74.09950653419286,
      76.6739327545555
     ],
     "B": [
      73.53694581918141,
      77.09486308430787
     ],
     "C": [
      74.09950653419286,
      76.6739327545555
     ]
    },
    "jevbench_score": 75.7463117322681,
    "measured_in": "v1.6.43",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 5,
    "ranks": {
     "A": 5,
     "B": 4,
     "C": 5,
     "capability": 16
    },
    "scores": {
     "A": 75.7463117322681,
     "B": 75.74847016866207,
     "C": 75.7463117322681
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.4687188219977543,
     "p50_s_raw": 0.4687188219977543,
     "p95_s_adjusted": 0.682831523841014,
     "p95_s_raw": 0.682831523841014
    },
    "views": {
     "Balanced 33:33:33": 72.3717813701936,
     "Emphasis on Accuracy 60:20:20": 73.68894327115758,
     "Emphasis on Cost 20:20:60": 67.19182812608965,
     "Emphasis on Speed 20:60:20": 76.92721011051661,
     "Intelligence only": 75.75710514449688
    }
   },
   "rank": 12,
   "ranked": true,
   "ranks": {
    "A": 12,
    "B": 8,
    "C": 11,
    "capability": 20
   },
   "repo": "https://subq.ai",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 75.73702535215112,
    "B": 75.74104045901554,
    "C": 75.73702535215112
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4690868764882907,
    "p50_s_raw": 0.4690868764882907,
    "p95_s_adjusted": 0.6896718636329751,
    "p95_s_raw": 0.6896718636329751
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Hosted SubQ Snap 1.1; measured 22 Oct 2026. SubQ Snap 1.1 is a capacity-limited public preview (first come first served, with a waitlist). SAME-AUTHOR SECOND use of this draw: SubQ already saw the sealed items on 13 Oct 2026 through subq-snap-preview; version 1.1 was released afterwards. The old preview name now routes to the same model; its historical row remains labelled as the 13 Oct measurement, with its frozen USD 0.05/M tariff unchanged. This row uses the published USD 0.04/M input tariff, output free, excluding signup credits. All 1,500 items answered once with native probabilities and reported token usage. Permission-error requests never reached the model; collection resumed after access was restored. S/P check versus the preview: sealed accuracy 84.25% vs 85.00% (-0.75 pp), public 90.33% vs 88.67% (+1.67 pp); sealed-minus-public gain -2.42 pp, 95% CI [-6.00, +1.17] pp, zero inside: not flagged. No evidence of public-item tuning in this check; prior exposure remains disclosed.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 85.59462369626596,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 65.11720442213118
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 85.59462369626596
      },
      "calibration_gen": {
       "accuracy": 0.7722772277227723,
       "cc": 61.34400703600102,
       "cc_by_type": {
        "choice": 68.18663838812301,
        "noul": 43.99999999999999,
        "score": 71.84538271988004
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.34400703600102
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 90.32923568776323,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 100,
        "score": 77.4917721039401
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 90.32923568776323
      },
      "fairness": {
       "accuracy": 0.8958333333333334,
       "cc": 83.27632603982205,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 89.82897811946613
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.27632603982205
      },
      "jaggedness": {
       "accuracy": 0.7333333333333333,
       "cc": 59.79508259236162,
       "cc_by_type": {
        "choice": 65.35129932627527,
        "noul": 40.54054054054055,
        "score": 73.49340791026906
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 59.79508259236162
      },
      "llm_router": {
       "accuracy": 0.9041533546325878,
       "cc": 83.1733277544028,
       "cc_by_type": {
        "choice": 92.86783481553225,
        "noul": 97.93814432989691,
        "score": 58.714004117779254
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 83.1733277544028
      },
      "long_state": {
       "accuracy": 0.5333333333333333,
       "cc": -8.212860511185264,
       "cc_by_type": {
        "choice": 48.387096774193544,
        "noul": -100,
        "score": 26.97432169225067
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 45.07843930656005,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": 0,
        "score": 59.977585960917246
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 45.07843930656005
      },
      "multilingual": {
       "accuracy": 0.8109090909090909,
       "cc": 68.1315293010611,
       "cc_by_type": {
        "choice": 82.3932181284643,
        "noul": 55.55555555555556,
        "score": 66.44581421916344
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 68.1315293010611
      },
      "negation_pair_gen": {
       "accuracy": 0.9069767441860465,
       "cc": 78.40427254862233,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 68.54615097920032
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 78.40427254862233
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 77.85381509880871,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 100,
        "score": 51.57946331444412
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 77.85381509880871
      },
      "usecase": {
       "accuracy": 0.8090909090909091,
       "cc": 67.41426811771518,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": 54.16666666666667,
        "score": 68.9094710198122
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 67.41426811771518
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 76.87644537854416,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": 49.676955183251536
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 76.87644537854416
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 52.01247549696391,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 43.44142034649848
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 52.01247549696391
      },
      "de": {
       "accuracy": 0.9375,
       "cc": 91.23772581977327,
       "cc_by_type": {
        "choice": 92.03539823008849,
        "noul": 100,
        "score": 81.67777922923129
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 91.23772581977327
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.71362792645883,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.9937701853355
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.71362792645883
      },
      "en": {
       "accuracy": 0.8462206776715899,
       "cc": 75.36424883571205,
       "cc_by_type": {
        "choice": 83.52473120904064,
        "noul": 71.22302158273382,
        "score": 71.34499371536167
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 75.36424883571205
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 74.6441100390774,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 23.932330117232226
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 74.6441100390774
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.54067102595756,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 99.075564990441
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.54067102595756
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 84.32987213230238,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 95.34255757337776
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 84.32987213230238
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 47.270696258480406,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 0,
        "score": 41.812088775441225
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 47.270696258480406
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 60.8886522963651,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -3.304425028285829
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 60.8886522963651
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 38.90739241902529,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 39.99999999999999,
        "score": -8.980761583273612
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 38.90739241902529
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 24.116527114148045,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 72.34958134244413
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 24.116527114148045
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 77.61321393008984,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.50630845693618
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 77.61321393008984
      },
      "nl": {
       "accuracy": 0.7,
       "cc": -2.0136452061382903,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": -72.70760228508153
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8974358974358975,
       "cc": 82.55541849674057,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80,
        "score": 75.28369095361572
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 82.55541849674057
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 76.47339934882655,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 61.24105838825926
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 76.47339934882655
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 67.90107670920433,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 69.23514502122998
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 67.90107670920433
      },
      "tr": {
       "accuracy": 1,
       "cc": 94.95103283164401,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 84.85309849493206
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 94.95103283164401
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 76.94915988090104,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 95.36360867496114
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 76.94915988090104
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 47.08691030844218,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 7.927397591993202
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 47.08691030844218
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8733333333333333,
       "cc": 83.63918176773377,
       "cc_by_type": {
        "choice": 83.63918176773377
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 83.63918176773377
      },
      "noul": {
       "accuracy": 0.84,
       "cc": 68,
       "cc_by_type": {
        "noul": 68
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 68
      },
      "score": {
       "accuracy": 0.7653333333333333,
       "cc": 68.99136919378957,
       "cc_by_type": {
        "score": 68.99136919378957
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 68.99136919378957
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6363636363636364,
       "cc": 37.1147079515983,
       "cc_by_type": {
        "choice": 84,
        "noul": 0,
        "score": 27.3441238547949
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 37.1147079515983
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 79.64986035349233,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 61.1718032826992
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 79.64986035349233
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 66.10739036282742,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 100,
        "score": 30.842496291734278
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 66.10739036282742
      },
      "demand_forecasting": {
       "accuracy": 0.4117647058823529,
       "cc": -17.615704748395572,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -116.00500898202881
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 59.66475542632195,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": 33.33333333333333,
        "score": 74.64644019200932
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.66475542632195
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 59.13377530019597,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": 69.22522527165711
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 59.13377530019597
      },
      "financial_crime": {
       "accuracy": 0.8333333333333334,
       "cc": 57.10562696146352,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": 14.28571428571428,
        "score": 66.83508816730375
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 57.10562696146352
      },
      "gaming": {
       "accuracy": 0.9090909090909091,
       "cc": 77.68713959818707,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50,
        "score": 93.51438046703507
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 77.68713959818707
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 17.776309130734905,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -60,
        "score": 34.16226072553804
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 17.776309130734905
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 82.68459256985513,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 81.38711104289872
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 82.68459256985513
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 58.988451873179145,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 55.25992926294832
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 58.988451873179145
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 27.71488816719786,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 46.58552471664734
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 27.71488816719786
      },
      "llm_guardrails": {
       "accuracy": 0.9259259259259259,
       "cc": 91.33913402560951,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 74.01740207682855
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 91.33913402560951
      },
      "model_routing": {
       "accuracy": 0.8847926267281107,
       "cc": 79.3376077944341,
       "cc_by_type": {
        "choice": 96.49350649350649,
        "noul": 96.96969696969697,
        "score": 44.54961992009885
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.3376077944341
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 44.770304956543136,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": 0,
        "score": 68.85985920744133
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 44.770304956543136
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 63.17987181140378,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 72.91937402288359
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 63.17987181140378
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 38.55486178559623,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": 45.2212356030941
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 38.55486178559623
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 46.64142601531304,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": 33.33333333333333,
        "score": 79.0547128285478
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 46.64142601531304
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.814456390198,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.44336917059398
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.814456390198
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      84.27453802496953,
      90.10715200144764
     ],
     "I": [
      71.22059116474675,
      78.89225696940721
     ],
     "capability": [
      78.27586653706702,
      83.99171809392114
     ],
     "composite": {
      "A": [
       74.09061953247686,
       76.6644175462098
      ],
      "B": [
       73.52994358661283,
       77.08716692169163
      ],
      "C": [
       74.09061953247686,
       76.6644175462098
      ]
     },
     "seed": 78
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.06513871774217,
     "capability": 81.91112193111952,
     "intelligence": 75.75710514449688
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8733333333333333,
      "calibration": 88.47890657091898,
      "cc": 83.63918176773377,
      "cc_by_type": {
       "choice": 83.63918176773377
      },
      "cc_tiered": 85.19560621499613,
      "cc_tiered_open": 88.50761395257393,
      "cc_tiered_sealed": 81.88359847741832,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.63918176773377
     },
     "noul": {
      "accuracy": 0.84,
      "calibration": 85.38097448521003,
      "cc": 68,
      "cc_by_type": {
       "noul": 68
      },
      "cc_tiered": 72.3999892949187,
      "cc_tiered_open": 83.07729468599035,
      "cc_tiered_sealed": 61.722683903847056,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 68
     },
     "score": {
      "accuracy": 0.7653333333333333,
      "calibration": 90.33553509709749,
      "cc": 68.99136919378957,
      "cc_by_type": {
       "score": 68.99136919378957
      },
      "cc_tiered": 69.6757199235758,
      "cc_tiered_open": 72.0081553126909,
      "cc_tiered_sealed": 67.34328453446071,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.99136919378957
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 72.36047856736685,
    "Emphasis on Accuracy 60:20:20": 73.68191206349243,
    "Emphasis on Cost 20:20:60": 67.18598208536949,
    "Emphasis on Speed 20:60:20": 76.90422650524084,
    "Intelligence only": 75.75710514449688
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "SubQ",
   "axes": {
    "calibration": 90.69740052356492,
    "cost": 57.693900899055016,
    "intelligence": 72.25005247787018,
    "speed": 86.35826133612204
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04418888669165353,
      "ece_score": 91.16222266166929,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 91.16222266166929,
      "score_before_time_limit": 91.16222266166929,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.14400000000000002,
      "brier": 0.07100960878955642,
      "ece": 0.045594591092318275,
      "n": 375,
      "score": 90.88108178153634,
      "score_before_time_limit": 90.88108178153634,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9653333333333334,
      "interval90_width": 0.22466666666666665,
      "n": 375,
      "nrps": 0.047718109524649006,
      "score": 90.04889712748917,
      "score_before_time_limit": 90.04889712748917,
      "time_limit_failed_share": 0,
      "top_ece": 0.0756519739627838
     }
    },
    "score": 90.69740052356492
   },
   "capability": 81.47372650071756,
   "composite_ci95": {
    "A": [
     72.52502278899739,
     75.25211552563907
    ],
    "B": [
     71.43810693082546,
     75.36008043389832
    ],
    "C": [
     72.52502278899739,
     75.25211552563907
    ]
   },
   "cost": {
    "basis": "Operator's own published standard usage tariff: USD 0.05 per 1M INPUT tokens with no output billing. Two vendor sources, both snapshotted: the pricing section of https://subq.ai/ read on 14 Oct 2026 08:10Z ('In $0.05 per 1M tokens Input / Out Free Output') and the vendor's own launch post 'Introducing SubQ Snap' ('SubQ Snap is priced at $0.05 per million input tokens, with no charge for output tokens'), which also dates the tariff to the product's launch; Subquadratic's Alexander Whedon separately confirmed the figure in writing to the evaluator on 14 Oct 2026 01:16 Berlin (the published review). Counted under interpretation I-1 (a manufacturer's standard, non-promotional launch list price counts from day 1 - the reading that already admitted kushal-gemma4-31b-it-autoloops). DISCLOSED: Snap is in a capacity-limited public preview - first come, first served, with a waitlist once capacity is reached - so this is the vendor's stated usage price for a product that is not yet openly bookable by everyone; the published pricing interpretation states explicitly that I-1 covers this tariff during that public preview, on the condition that the capacity limit and waitlist are stated here, as they are. The same pricing section offers 'Free 100M tokens for the first 10,000 users': that is a promotional trial credit and is deliberately NOT used - a free tier is priced at the published paid tariff, never as cost 0. The tariff is multiplied by this row's OWN measured input tokens (1,943,705 over 1,500 answered rows, mean 1,296) and divided by the 1,485 items of the common cost basis the scorer uses. The output term is zero twice over: the tariff bills no output, and the vendor's own usage meter reports 0 output tokens on all 1,500 rows. Gate chain: the pricing update made the cost axis conditional on a separate frozen-pricing change passing re-freeze, unchanged-row proof and independent review; the pricing review requested it; the scoped change adds this one key to pricing_v15.LAUNCH_LIST_PRICE_API and nothing else (r113/pricing/CANDIDATE.diff; unchanged-row proof r113/receipts/PRICING-EQUIV-R113.json and two rounds of independent review, r113/review-pricing/REVIEW.json (REVISE) and r113/review-pricing-r2/REVIEW.json (PASS); the findings about this row's text and the builder's gates are applied here, the two about the publication layer are on the build sheet). INSIDE THE OFFICIAL COST CAP on the common basis: USD 0.025716 per 1,000 decisions is 0.398x the official cap USD 0.064595. Over all 1,500 items the row cost USD 0.064790 (1.003x the cap) and was disclosed as ranked over the cap in v1.6.29; that disclosure is superseded. COMMON COST BASIS (v1.6.30): cost per 1,000 decisions is computed on the common 1,485 items of this draw, leaving out the 15 very long items (at least 76,358 input tokens as counted by the rows that answered them). They are the 15 items the Jev reference refused with max_tokens_exceeded, which the other rows that did not answer them refused or failed on; their tokens count for no row, whether the row refused or answered them, and a refused item still counts as a wrong answer for Intelligence. This row: USD 0.025716 per 1,000 decisions (0.398x the official cost cap USD 0.064595), against USD 0.064790 over all 1,500 items (v1.6.29).",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.06479016666666666
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025715993265993265
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "SubQ Snap (preview, SubQ)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-13",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted api.subq.ai /v1/systemone API, model `subq-snap-preview`; latency measured from Sandy (Germany) on the official per-item latency subset without API adjustment. Full S1200+P300 draw of v1.6-regular-20261012 through the unchanged run_api_full_v161.py; the full-set exposure gate logged all 1,200 sealed items to provider `subq` before the first request (every sealed item of this draw was already retired). Native probabilities on all three question types, no generated tokens. Hosting implementation and deployed weights were not independently inspected. The measurement request (X, 12 Oct 2026). SubQ retains API inputs indefinitely by default; acceptable only because this draw's items are already retired.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 75.21805946266113,
    "I_sealed": 69.28204549307924,
    "base": 72.25005247787018,
    "excess": -3.903512800773818,
    "gap": 5.936013969581893,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 84.9873634422428,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 83.01352626612143,
       "judge": 80.22412656558998,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 73.1207729468599,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 67.54604199888068,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 64.92519654954472,
       "hard": 76.69157031955869,
       "judge": 72.3442858248807,
       "standard": 43.36804234319263
      }
     },
     "sealed|choice": {
      "cc": 84.72260612932492,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 78.66312015727996,
       "judge": 87.65915768854063,
       "standard": 88.60999789074035
      }
     },
     "sealed|noul": {
      "cc": 57.554016750105276,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 49.09090909090909,
       "judge": 60.00000000000001,
       "standard": 58.20895522388059
      }
     },
     "sealed|score": {
      "cc": 65.56951359980751,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 85.17961744525421,
       "hard": 67.30197547341396,
       "judge": 61.73385013050089,
       "standard": 58.05303313383112
      }
     }
    }
   },
   "jevbench_score": 74.37845073677168,
   "key": "subq-snap-preview",
   "last_measured_on": "2026-10-13",
   "licence": "Proprietary hosted API (no weights release)",
   "listing": "ranked",
   "measured_in": "v1.6.17",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:api.subq.ai/v1/systemone#subq-snap-preview",
   "noul_decisive": {
    "acc_among_decisive": 0.9532710280373832,
    "decisive": 321,
    "decisive_rate": 0.856,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 7,
    "axes": {
     "calibration": 90.69740052356492,
     "cost": 57.693900899055016,
     "intelligence": 72.25005247787018,
     "speed": 86.40460211903752
    },
    "board_release": "v1.6.43",
    "capability": 81.47372650071756,
    "capability_api_board_rank": 5,
    "capability_open_board_rank": null,
    "capability_rank": 10,
    "composite_ci95": {
     "A": [
      72.53319025366382,
      75.26090880321355
     ],
     "B": [
      71.44444637250206,
      75.36713509179089
     ],
     "C": [
      72.53319025366382,
      75.26090880321355
     ]
    },
    "jevbench_score": 74.38704101088614,
    "measured_in": "v1.6.17",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 8,
    "ranks": {
     "A": 8,
     "B": 8,
     "C": 8,
     "capability": 21
    },
    "scores": {
     "A": 74.38704101088614,
     "B": 73.94958963293669,
     "C": 74.38704101088614
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.46557116050098557,
     "p50_s_raw": 0.46557116050098557,
     "p95_s_adjusted": 0.4915341741005249,
     "p95_s_raw": 0.4915341741005249
    },
    "views": {
     "Balanced 33:33:33": 70.18014569964379,
     "Emphasis on Accuracy 60:20:20": 70.99371113720164,
     "Emphasis on Cost 20:20:60": 64.58876677881489,
     "Emphasis on Speed 20:60:20": 75.87938757645203,
     "Intelligence only": 72.25005247787018
    }
   },
   "rank": 28,
   "ranked": true,
   "ranks": {
    "A": 28,
    "B": 19,
    "C": 24,
    "capability": 26
   },
   "repo": "https://subq.ai",
   "repriced_in": "v1.6.20",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.37845073677168,
    "B": 73.94279784299546,
    "C": 74.37845073677168
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4660752070049057,
    "p50_s_raw": 0.4660752070049057,
    "p95_s_adjusted": 0.49626982034533285,
    "p95_s_raw": 0.49626982034533285
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 89.67339746240113,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 77.35352572053671
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 89.67339746240113
      },
      "calibration_gen": {
       "accuracy": 0.7524752475247525,
       "cc": 59.325989058477944,
       "cc_by_type": {
        "choice": 71.36797454931072,
        "noul": 36.00000000000001,
        "score": 70.60999262612313
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 59.325989058477944
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 89.73374806927508,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 100,
        "score": 75.70530924847564
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 89.73374806927508
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.11997769875624,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 95.35993309626873
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.11997769875624
      },
      "jaggedness": {
       "accuracy": 0.7833333333333333,
       "cc": 60.15465710190771,
       "cc_by_type": {
        "choice": 78.34456207892205,
        "noul": 29.729729729729737,
        "score": 72.38967949707134
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 60.15465710190771
      },
      "llm_router": {
       "accuracy": 0.9073482428115016,
       "cc": 80.88165044070985,
       "cc_by_type": {
        "choice": 90.49044642070969,
        "noul": 95.8762886597938,
        "score": 56.278216241626076
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 80.88165044070985
      },
      "long_state": {
       "accuracy": 0.4666666666666667,
       "cc": -25.521900617988198,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -12.049572821706533
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 33.822823442121184,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": -25,
        "score": 51.21073836760064
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 33.822823442121184
      },
      "multilingual": {
       "accuracy": 0.7745454545454545,
       "cc": 63.63301054731818,
       "cc_by_type": {
        "choice": 79.45875448320835,
        "noul": 50,
        "score": 61.44027715874617
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 63.63301054731818
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 77.29805491838812,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 73.33333333333334,
        "score": 65.22749808849767
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 77.29805491838812
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 84.09170943563231,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 64.28714031890895
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 84.09170943563231
      },
      "usecase": {
       "accuracy": 0.7909090909090909,
       "cc": 64.3360385766533,
       "cc_by_type": {
        "choice": 85.29411764705883,
        "noul": 41.66666666666667,
        "score": 66.04733141623439
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 64.3360385766533
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 61.46987986833912,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100,
        "score": 22.504877700255456
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 61.46987986833912
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 42.044641118681675,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 13.537917211651784
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 42.044641118681675
      },
      "de": {
       "accuracy": 0.875,
       "cc": 84.30873412452327,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 76.82000768330435
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 84.30873412452327
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 66.17191426416157,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 98.7019625317771
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 66.17191426416157
      },
      "en": {
       "accuracy": 0.8523023457862728,
       "cc": 74.39868278842214,
       "cc_by_type": {
        "choice": 85.94110396504801,
        "noul": 66.90647482014388,
        "score": 70.34846958007451
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 74.39868278842214
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 68.10721678264782,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 14.10957856980316
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 68.10721678264782
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 65.95984983965347,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 97.33310143152872
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.95984983965347
      },
      "fr": {
       "accuracy": 1,
       "cc": 98.39208642380281,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.17625927140838
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 98.39208642380281
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 51.02919425601916,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 0,
        "score": 53.08758276805747
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 51.02919425601916
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 67.3748258425729,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 16.15409561033757
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 67.3748258425729
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 39.17673354531658,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 19.999999999999996,
        "score": 11.827261795600252
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 39.17673354531658
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 18.81407201511874,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 56.44221604535622
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 18.81407201511874
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.79923075613463,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 97.06435893507054
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.79923075613463
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 14.038150782601484,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 1.4218116551637516
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 14.038150782601484
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 71.56044111873955,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 62.298758819612644
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 71.56044111873955
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 62.14187116919363,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 58.246473849360505
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 62.14187116919363
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 54.783703729457464,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 55.41494097560644
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 54.783703729457464
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 75.33063049195334,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 25.99189147586003
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 75.33063049195334
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 85.2525051117384,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 88.01557985134423
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 85.2525051117384
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 74.88775899012883,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 24.663276970386516
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 74.88775899012883
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8826666666666667,
       "cc": 84.84471574274289,
       "cc_by_type": {
        "choice": 84.84471574274289
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.84471574274289
      },
      "noul": {
       "accuracy": 0.816,
       "cc": 63.19999999999999,
       "cc_by_type": {
        "noul": 63.19999999999999
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 63.19999999999999
      },
      "score": {
       "accuracy": 0.7573333333333333,
       "cc": 66.8697791370952,
       "cc_by_type": {
        "score": 66.8697791370952
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 66.8697791370952
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 24.729183321197823,
       "cc_by_type": {
        "choice": 84,
        "noul": -50,
        "score": 40.18754996359347
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 24.729183321197823
      },
      "code_linting": {
       "accuracy": 0.9583333333333334,
       "cc": 91.60263751131986,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 74.80791253395952
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 91.60263751131986
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 33.127488956097665,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": 48.16295467317104
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 33.127488956097665
      },
      "demand_forecasting": {
       "accuracy": 0.5294117647058824,
       "cc": 15.378424713885506,
       "cc_by_type": {
        "choice": 53.21637426900585,
        "noul": 33.33333333333333,
        "score": -40.41443346068265
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 15.378424713885506
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 36.699498608592535,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 72.41733640548773
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 36.699498608592535
      },
      "feature_extraction": {
       "accuracy": 0.6842105263157895,
       "cc": -31.131294032996152,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 0,
        "score": -180.8152657467872
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.7,
       "cc": 56.44044459277766,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 14.28571428571428,
        "score": 55.03561949261869
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 56.44044459277766
      },
      "gaming": {
       "accuracy": 0.7272727272727273,
       "cc": 62.56265622049958,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 50,
        "score": 58.59389200644648
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 62.56265622049958
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 22.430603549022365,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -60,
        "score": 68.95847731373377
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 22.430603549022365
      },
      "knowledge_graphs": {
       "accuracy": 0.6818181818181818,
       "cc": 61.40956000739754,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 66.66666666666667,
        "score": 56.105910571800045
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 61.40956000739754
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 58.437908295955914,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 41.98039155453445
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 58.437908295955914
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 45.85644368492868,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100,
        "score": 34.34352460317314
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 45.85644368492868
      },
      "llm_guardrails": {
       "accuracy": 0.9259259259259259,
       "cc": 89.66786126407828,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 69.00358379223486
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 89.66786126407828
      },
      "model_routing": {
       "accuracy": 0.9032258064516129,
       "cc": 79.04214957440841,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 100,
        "score": 42.97060456738109
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.04214957440841
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 54.225709232648235,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 85.70975725648596
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 54.225709232648235
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 55.27714970469024,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 65.92429962474853
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 55.27714970469024
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 38.09260541997223,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": 29.056141383069402
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 38.09260541997223
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 55.326789380349254,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 33.33333333333333,
        "score": 61.63254205409125
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 55.326789380349254
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.91541807814257,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.74625423442768
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.91541807814257
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.01486620759374,
      91.38185522285914
     ],
     "I": [
      67.16649175722978,
      76.03035129242522
     ],
     "capability": [
      77.48528010616971,
      83.12552326478638
     ],
     "composite": {
      "A": [
       72.52502278899739,
       75.25211552563907
      ],
      "B": [
       71.43810693082546,
       75.36008043389832
      ],
      "C": [
       72.52502278899739,
       75.25211552563907
      ]
     },
     "seed": 74
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.69740052356492,
     "capability": 81.47372650071756,
     "intelligence": 72.25005247787018
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8826666666666667,
      "calibration": 91.16222266166929,
      "cc": 84.84471574274289,
      "cc_by_type": {
       "choice": 84.84471574274289
      },
      "cc_tiered": 84.85498478578386,
      "cc_tiered_open": 84.9873634422428,
      "cc_tiered_sealed": 84.72260612932492,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.84471574274289
     },
     "noul": {
      "accuracy": 0.816,
      "calibration": 90.88108178153634,
      "cc": 63.19999999999999,
      "cc_by_type": {
       "noul": 63.19999999999999
      },
      "cc_tiered": 65.33739484848259,
      "cc_tiered_open": 73.1207729468599,
      "cc_tiered_sealed": 57.554016750105276,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 63.19999999999999
     },
     "score": {
      "accuracy": 0.7573333333333333,
      "calibration": 90.04889712748917,
      "cc": 66.8697791370952,
      "cc_by_type": {
       "score": 66.8697791370952
      },
      "cc_tiered": 66.5577777993441,
      "cc_tiered_open": 67.54604199888068,
      "cc_tiered_sealed": 65.56951359980751,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 66.8697791370952
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.1699511795255,
    "Emphasis on Accuracy 60:20:20": 70.98745142939559,
    "Emphasis on Cost 20:20:60": 64.58358556228033,
    "Emphasis on Speed 20:60:20": 75.85793898837869,
    "Intelligence only": 72.25005247787018
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Surogate",
   "axes": {
    "calibration": 90.88042351612289,
    "cost": 71.03441945513822,
    "intelligence": 56.055735926690254,
    "speed": 86.53673235812263
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05932792591743529,
      "ece_score": 88.13441481651294,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 88.13441481651294,
      "score_before_time_limit": 88.13441481651294,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2695417789757413,
      "brier": 0.08438373400978966,
      "ece": 0.039029372562851536,
      "n": 371,
      "score": 92.19412548742969,
      "score_before_time_limit": 92.19412548742969,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.967479674796748,
      "interval90_width": 0.37466124661246614,
      "n": 369,
      "nrps": 0.0647012985012109,
      "score": 92.31273024442605,
      "score_before_time_limit": 92.31273024442605,
      "time_limit_failed_share": 0,
      "top_ece": 0.04452204830513414
     }
    },
    "score": 90.88042351612289
   },
   "capability": 73.46807972140657,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 780,723 input + 1,477 output tokens over 1,477 decisions = 529.587000677 tokens/decision. GPU-time cost USD 0.00923675345058052/1,000; old list-price cost USD 0.0478728300609343/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00923675345058052/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.04713878
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009236753450580516
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Surogate Rune 26B-A4B v3 (v1.6 pool)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB, a thin transport shim around the author's engine in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 58.17716696493766,
    "I_sealed": 53.934304888442846,
    "base": 56.055735926690254,
    "excess": 1.6694744340565726,
    "gap": 4.242862076494816,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.95733901606096,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 66.02705253224286,
       "judge": 80.22412656558997,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 42.59075224292616,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 47.82608695652173,
       "judge": 19.999999999999996,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 55.98340963582588,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 62.28084236876889,
       "hard": 59.46587194956875,
       "judge": 61.146761005919444,
       "standard": 38.12474158672826
      }
     },
     "sealed|choice": {
      "cc": 73.06247198507991,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 84.03665656640308,
       "hard": 68.94796380090497,
       "judge": 73.9292364990689,
       "standard": 74.50424929178472
      }
     },
     "sealed|noul": {
      "cc": 34.31372825609765,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 68,
       "hard": 16.50485436893203,
       "judge": 36.8421052631579,
       "standard": 49.29577464788733
      }
     },
     "sealed|score": {
      "cc": 54.426714424150994,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 71.74170476496207,
       "hard": 48.52758483569058,
       "judge": 67.30550244555229,
       "standard": 38.24929639856435
      }
     }
    }
   },
   "jevbench_score": 73.42639028852348,
   "key": "surogate-rune-26b-a4b-v3-v16",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "surogate/rune-26b-a4b-GGUF@bd4a7cbbed66af1a95dfbac12cabf52e6e677411",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.966789667896679,
    "decisive_rate": 0.7304582210242587,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.88042351612289,
     "cost": 49.597326937281316,
     "intelligence": 56.055735926690254,
     "speed": 86.95393114544444
    },
    "board_release": "v1.6.43",
    "capability": 73.46807972140657,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 15,
    "capability_rank": 31,
    "composite_ci95": {
     "A": [
      62.82778837763545,
      66.19023981114698
     ],
     "B": [
      59.77310724201828,
      64.70527953464652
     ],
     "C": [
      45.16787327358027,
      66.15052716337196
     ]
    },
    "jevbench_score": 65.0490738113794,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 27,
    "rank": 48,
    "ranks": {
     "A": 48,
     "B": 47,
     "C": 45,
     "capability": 55
    },
    "scores": {
     "A": 65.0490738113794,
     "B": 62.796501415524304,
     "C": 56.77782262023472
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3823961119982414,
     "p50_s_raw": 0.1161980559991207,
     "p95_s_adjusted": 0.5273432816902641,
     "p95_s_raw": 0.18867164084513205
    },
    "views": {
     "Balanced 33:33:33": 59.63127588509936,
     "Emphasis on Accuracy 60:20:20": 57.75697016597956,
     "Emphasis on Cost 20:20:60": 54.76969389057124,
     "Emphasis on Speed 20:60:20": 67.8565544623977,
     "Intelligence only": 55.15648620822009
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card metadata",
    "kind": "v1.6.1 addendum a7 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 41,
   "ranked": true,
   "ranks": {
    "A": 41,
    "B": 62,
    "C": 74,
    "capability": 72
   },
   "repo": "https://huggingface.co/surogate/rune-26b-a4b-GGUF",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 73.42639028852348,
    "B": 69.141265811278,
    "C": 64.08992963580992
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3835699379909784,
    "p50_s_raw": 0.11678496899548918,
    "p95_s_adjusted": 0.5787382701702881,
    "p95_s_raw": 0.21436913508514407
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-26B-A4B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.71263475548702,
      91.20016913118003
     ],
     "I": [
      50.88578039677805,
      60.44466553566163
     ],
     "capability": [
      69.30942731571565,
      74.85354775896664
     ],
     "composite": null,
     "seed": 116
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.88042351612289,
     "capability": 73.46807972140657,
     "intelligence": 56.055735926690254
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8,
      "calibration": 88.13441481651294,
      "cc": 74.4228564932789,
      "cc_by_type": {
       "choice": 74.4228564932789
      },
      "cc_tiered": 74.50990550057043,
      "cc_tiered_open": 75.95733901606096,
      "cc_tiered_sealed": 73.06247198507991,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 74.4228564932789
     },
     "noul": {
      "accuracy": 0.6986666666666667,
      "calibration": 92.19412548742969,
      "cc": 39.733333333333334,
      "cc_by_type": {
       "noul": 39.733333333333334
      },
      "cc_tiered": 38.4522402495119,
      "cc_tiered_open": 42.59075224292616,
      "cc_tiered_sealed": 34.31372825609765,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.733333333333334
     },
     "score": {
      "accuracy": 0.6666666666666666,
      "calibration": 92.31273024442605,
      "cc": 54.21453874970139,
      "cc_by_type": {
       "score": 54.21453874970139
      },
      "cc_tiered": 55.20506202998844,
      "cc_tiered_open": 55.98340963582588,
      "cc_tiered_sealed": 54.426714424150994,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 54.21453874970139
     }
    },
    "raw_on_A": {
     "calibration": 89.39331463026028,
     "capability": 74.65140006965763,
     "intelligence": 59.909485509054974
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.00857780001692,
    "Emphasis on Accuracy 60:20:20": 63.169891681110684,
    "Emphasis on Cost 20:20:60": 69.80488838696964,
    "Emphasis on Speed 20:60:20": 75.0926311699783,
    "Intelligence only": 56.05573592669025
   }
  },
  {
   "adapter_id": "images/swanone/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "blockbrain",
   "axes": {
    "calibration": 88.58151013990448,
    "cost": 71.43288940092707,
    "intelligence": 56.15104294255194,
    "speed": 82.65355535838808
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03565851721103598,
      "ece_score": 92.8682965577928,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 92.8682965577928,
      "score_before_time_limit": 92.8682965577928,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.18867924528301883,
      "brier": 0.10222826353990752,
      "ece": 0.07688828024001365,
      "n": 371,
      "score": 84.62234395199727,
      "score_before_time_limit": 84.62234395199727,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.924119241192412,
      "interval90_width": 0.2818428184281841,
      "n": 369,
      "nrps": 0.07387893374911293,
      "score": 88.25388990992337,
      "score_before_time_limit": 88.25388990992337,
      "time_limit_failed_share": 0,
      "top_ece": 0.08052163402620996
     }
    },
    "score": 88.58151013990448
   },
   "capability": 72.36627654122822,
   "class": "system-one-open",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 912,828 input + 1,624 output tokens over 1,624 decisions = 563.086206897 tokens/decision. GPU-time cost USD 0.00895853551980716/1,000; old list-price cost USD 0.0847829310344827/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00895853551980716/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.008958535519807157
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "swanOne (blockbrain, Qwen3.8-Flash-Next NVFP4)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTXPRO6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTXPRO6000",
   "intelligence": {
    "I_open": 56.23071661013901,
    "I_sealed": 56.07136927496486,
    "base": 56.15104294255194,
    "excess": -2.41404030726409,
    "gap": 0.15934733517415367,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 76.36851516458754,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 69.80182447310474,
       "judge": 84.17930125247197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 42.89026915113871,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 39.13043478260869,
       "judge": 39.99999999999999,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 49.43336551469081,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 57.945051339203516,
       "hard": 45.879570270586335,
       "judge": 54.70148526262065,
       "standard": 44.382933468748654
      }
     },
     "sealed|choice": {
      "cc": 69.57776531270856,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 74.72470623013821,
       "hard": 68.43891402714931,
       "judge": 64.61824953445065,
       "standard": 76.7212710924991
      }
     },
     "sealed|noul": {
      "cc": 45.48583128818902,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 84.00000000000001,
       "hard": 18.446601941747566,
       "judge": 60.52631578947369,
       "standard": 57.746478873239425
      }
     },
     "sealed|score": {
      "cc": 53.150511223997015,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 70.92864532337575,
       "hard": 47.07089246791437,
       "judge": 55.79271749917136,
       "standard": 52.457372273711435
      }
     }
    }
   },
   "jevbench_score": 72.46902913443873,
   "key": "swanone",
   "last_measured_on": "2026-10-02",
   "licence": "patches Apache-2.0, shim MIT; weights under the Qwen licence (LICENSE-NOTICE.md)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9269102990033222,
    "decisive_rate": 0.8113207547169812,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 88.58151013990448,
     "cost": 42.15074720483203,
     "intelligence": 56.15104294255194,
     "speed": 82.64878475169937
    },
    "board_release": "v1.6.43",
    "capability": 72.36627654122822,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      42.00000021718559,
      44.674228699218794
     ],
     "B": [
      40.50170443761996,
      44.404221038840014
     ],
     "C": [
      29.16666681748999,
      44.674228699218794
     ]
    },
    "jevbench_score": 43.78640157551217,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 47,
    "rank": 72,
    "ranks": {
     "A": 72,
     "B": 71,
     "C": 68,
     "capability": 62
    },
    "scores": {
     "A": 43.78640157551217,
     "B": 42.950891870154436,
     "C": 38.348855694306984
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.7114971062168479,
     "p50_s_raw": 0.28074855310842395,
     "p95_s_adjusted": 0.7637450117617846,
     "p95_s_raw": 0.30687250588089227
    },
    "views": {
     "Balanced 33:33:33": 39.75216183669048,
     "Emphasis on Accuracy 60:20:20": 39.813195135198455,
     "Emphasis on Cost 20:20:60": 35.153484373145204,
     "Emphasis on Speed 20:60:20": 45.65457340538371,
     "Intelligence only": 39.90509729608976
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 51,
   "ranked": true,
   "ranks": {
    "A": 51,
    "B": 66,
    "C": 76,
    "capability": 79
   },
   "repo": "https://github.com/blockbrain-ai/swanone-recipe",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 72.46902913443873,
    "B": 68.48836506526615,
    "C": 63.46957595477202
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.7102613491937518,
    "p50_s_raw": 0.2801306745968759,
    "p95_s_adjusted": 0.7642338716425001,
    "p95_s_raw": 0.30711693582125005
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      83.47295009833218,
      89.28636150565644
     ],
     "I": [
      49.957046630174666,
      60.997721736085786
     ],
     "capability": [
      67.47509953288571,
      74.36120880664787
     ],
     "composite": null,
     "seed": 104
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.58151013990448,
     "capability": 72.36627654122822,
     "intelligence": 56.15104294255194
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7786666666666666,
      "calibration": 92.8682965577928,
      "cc": 71.69462785256195,
      "cc_by_type": {
       "choice": 71.69462785256195
      },
      "cc_tiered": 72.97314023864806,
      "cc_tiered_open": 76.36851516458754,
      "cc_tiered_sealed": 69.57776531270856,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 71.69462785256195
     },
     "noul": {
      "accuracy": 0.744,
      "calibration": 84.62234395199727,
      "cc": 48.8,
      "cc_by_type": {
       "noul": 48.8
      },
      "cc_tiered": 44.188050219663864,
      "cc_tiered_open": 42.89026915113871,
      "cc_tiered_sealed": 45.48583128818902,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 48.8
     },
     "score": {
      "accuracy": 0.6506666666666666,
      "calibration": 88.25388990992337,
      "cc": 52.763382802943624,
      "cc_by_type": {
       "score": 52.763382802943624
      },
      "cc_tiered": 51.29193836934391,
      "cc_tiered_open": 49.43336551469081,
      "cc_tiered_sealed": 53.150511223997015,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.763382802943624
     }
    },
    "raw_on_A": {
     "calibration": 88.23312364196306,
     "capability": 74.40197927194893,
     "intelligence": 60.5708349019348
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.32630495841394,
    "Emphasis on Accuracy 60:20:20": 62.873174716632015,
    "Emphasis on Cost 20:20:60": 69.53593973812187,
    "Emphasis on Speed 20:60:20": 73.4167588826543,
    "Intelligence only": 56.15104294255194
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 600,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "mithalouni",
   "axes": {
    "calibration": 75.94,
    "cost": 68.33,
    "intelligence": 27.94,
    "speed": 79.18
   },
   "calibration": {
    "score": 75.94
   },
   "capability": 51.94,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     7.72,
     36.08
    ],
    "B": [
     6.26,
     32.42
    ],
    "C": [
     5.36,
     25.06
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE: v1.5 measured-input proxy; no exact base-model floor applies",
    "kind": "estimate",
    "usd_per_1000": 0.011367487684729064
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "system-one-open (Gemma 4 E2B LoRA on an L4)",
   "endpoint_kind": "demo",
   "intelligence": {
    "I_open": 27.92,
    "I_sealed": 22.82,
    "base": 27.94,
    "per_type_split": {
     "open|choice": {
      "cc": 56.8437,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 53.2973,
       "hard": 47.1532,
       "judge": 72.3138,
       "standard": 54.7928
      }
     },
     "open|noul": {
      "cc": 3.8026,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 42.8571,
       "hard": 4.3478,
       "judge": 0,
       "standard": -11.1111
      }
     },
     "open|score": {
      "cc": 23.1119,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.1386,
       "hard": 15.9484,
       "judge": 27.2824,
       "standard": 18.1699
      }
     },
     "sealed|choice": {
      "cc": 47.7778,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 72.1965,
       "hard": 39.8901,
       "judge": 59.2965,
       "standard": 34.0659
      }
     },
     "sealed|noul": {
      "cc": 9.754,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 41.1765,
       "hard": 9.0909,
       "judge": -10,
       "standard": 25
      }
     },
     "sealed|score": {
      "cc": 10.9164,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 11.5419,
       "hard": 7.5012,
       "judge": -9.576,
       "standard": 48.1724
      }
     }
    }
   },
   "jevbench_score": 16.39,
   "key": "system-one-open",
   "last_measured_on": "2026-10-06",
   "licence": "MIT (repository LICENSE; Gemma weights keep Google’s terms)",
   "listing": "ranked",
   "method": "v1.7",
   "open": "yes",
   "previous": {
    "api_board_rank": 38,
    "axes": {
     "calibration": 75.94,
     "cost": 68.33,
     "intelligence": 27.94,
     "speed": 79.32
    },
    "board_release": "v1.6.43",
    "capability": 51.94,
    "capability_api_board_rank": 26,
    "capability_open_board_rank": null,
    "capability_rank": 116,
    "composite_ci95": {
     "A": [
      7.72,
      36.09
     ],
     "B": [
      6.26,
      32.43
     ],
     "C": [
      5.36,
      25.06
     ]
    },
    "jevbench_score": 16.39,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 152,
    "ranks": {
     "A": 152,
     "B": 154,
     "C": 152,
     "capability": 171
    },
    "scores": {
     "A": 16.39,
     "B": 13.94,
     "C": 11.38
    },
    "speed": {
     "adjustment": "x2 (demo)",
     "n": 600,
     "p50_s_adjusted": 1.02,
     "p50_s_raw": 0.51,
     "p95_s_adjusted": 1.146,
     "p95_s_raw": 0.573
    },
    "views": null
   },
   "rank": 198,
   "ranked": true,
   "ranks": {
    "A": 198,
    "B": 198,
    "C": 198,
    "capability": 208
   },
   "repo": "https://github.com/mithalouni/system-one-open",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 16.39,
    "B": 13.94,
    "C": 11.38
   },
   "speed": {
    "adjustment": "x2 (demo)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 1.028,
    "p50_s_raw": 0.514,
    "p95_s_adjusted": 1.174,
    "p95_s_raw": 0.587
   },
   "status": {
    "answered_ok": 600,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "1059a8be40a52bc4482c5bd40a45a368b53c2849acb4c370bc8735079c33e5a3"
   }
  },
  {
   "adapter_id": "images/system-one-sg/spec.json:remote_inproc_v15.RemoteInprocV15Adapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Sean Goedecke",
   "axes": {
    "calibration": 30.16473321941294,
    "cost": 68.27616759897502,
    "intelligence": 29.271508779158296,
    "speed": 90.51961611379383
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.5207165079484644,
      "ece_score": 0,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 0,
      "score_before_time_limit": 0,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.07547169811320753,
      "brier": 0.26938609640716954,
      "ece": 0.26647263490315864,
      "n": 371,
      "score": 46.70547301936827,
      "score_before_time_limit": 46.70547301936827,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.5718157181571816,
      "interval90_width": 0.08897922312556458,
      "n": 369,
      "nrps": 0.18780642533502573,
      "score": 43.78872663887056,
      "score_before_time_limit": 43.78872663887056,
      "time_limit_failed_share": 0,
      "top_ece": 0.46820952094378154
     }
    },
    "score": 30.16473321941294
   },
   "capability": 29.718120999285617,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 8B active parameters; throughput 30762.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 941,626 input + 1,624 output tokens over 1,624 decisions = 580.818965517 tokens/decision. GPU-time cost USD 0.01141461006257/1,000; old list-price cost USD 0.0682938189655172/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.01141461006257/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011414610062569962
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "system-one (Qwen3-8B, Sean Goedecke)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 32.67027386076885,
    "I_sealed": 25.87274369754774,
    "base": 29.271508779158296,
    "excess": 4.224142520782868,
    "gap": 6.797530163221111,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 45.21508118936821,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 30.16671909405474,
       "judge": 48.58272907053395,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 28.967563837129056,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 30.434782608695656,
       "judge": 19.999999999999996,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 23.8281765558093,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 65.82115115650328,
       "hard": 4.701913776879685,
       "judge": 35.713306286008326,
       "standard": 23.25652021802299
      }
     },
     "sealed|choice": {
      "cc": 35.384978952910906,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 54.77052693814206,
       "hard": 19.57013574660632,
       "judge": 35.7541899441341,
       "standard": 56.76807488606972
      }
     },
     "sealed|noul": {
      "cc": 28.751778735776956,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72,
       "hard": -2.9126213592232997,
       "judge": 44.736842105263165,
       "standard": 46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 13.48147340395537,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 53.14233360647993,
       "hard": 0.26200782926150135,
       "judge": 16.1186769737661,
       "standard": 16.134169097364737
      }
     }
    }
   },
   "jevbench_score": 14.739752382089495,
   "key": "system-one-sg",
   "last_measured_on": "2026-10-02",
   "licence": "no licence file in the repository as of 19 Sep; Qwen3 weights Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.7288629737609329,
    "decisive_rate": 0.9245283018867925,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 30.16473321941294,
     "cost": 44.96855803039237,
     "intelligence": 29.271508779158296,
     "speed": 90.8867694864598
    },
    "board_release": "v1.6.43",
    "capability": 29.718120999285617,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      5.879902468266102,
      15.68532110290842
     ],
     "B": [
      5.313278874085685,
      14.848330476734906
     ],
     "C": [
      4.08326560296257,
      10.892584099241958
     ]
    },
    "jevbench_score": 11.027723483913586,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 128,
    "rank": 167,
    "ranks": {
     "A": 167,
     "B": 167,
     "C": 168,
     "capability": 230
    },
    "scores": {
     "A": 11.027723483913586,
     "B": 10.28901604793032,
     "C": 7.658141308273324
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24992560098180547,
     "p50_s_raw": 0.04996280049090274,
     "p95_s_adjusted": 0.3262212941306643,
     "p95_s_raw": 0.08811064706533216
    },
    "views": {
     "Balanced 33:33:33": 12.338652673779826,
     "Emphasis on Accuracy 60:20:20": 10.21232996468921,
     "Emphasis on Cost 20:20:60": 12.38939107483353,
     "Emphasis on Speed 20:60:20": 15.503079386401165,
     "Intelligence only": 8.114714247763947
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 207,
   "ranked": true,
   "ranks": {
    "A": 207,
    "B": 204,
    "C": 207,
    "capability": 271
   },
   "repo": "https://github.com/sgoedecke/system-one",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 14.739752382089495,
    "B": 13.475119928096834,
    "C": 10.235939154228817
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2562555870157667,
    "p50_s_raw": 0.053127793507883325,
    "p95_s_adjusted": 0.3462302790256217,
    "p95_s_raw": 0.09811513951281085
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3-8B (frozen, BF16), the model of the author's demos",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      26.715281574276638,
      33.82082873066771
     ],
     "I": [
      22.79600693038093,
      33.62231882810145
     ],
     "capability": [
      25.123779554066562,
      33.19293706149128
     ],
     "composite": null,
     "seed": 105
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 30.16473321941294,
     "capability": 29.718120999285617,
     "intelligence": 29.271508779158296
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5133333333333333,
      "calibration": 0,
      "cc": 37.76228413364529,
      "cc_by_type": {
       "choice": 37.76228413364529
      },
      "cc_tiered": 40.30003007113956,
      "cc_tiered_open": 45.21508118936821,
      "cc_tiered_sealed": 35.384978952910906,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 37.76228413364529
     },
     "noul": {
      "accuracy": 0.6666666666666666,
      "calibration": 46.70547301936827,
      "cc": 33.33333333333333,
      "cc_by_type": {
       "noul": 33.33333333333333
      },
      "cc_tiered": 28.859671286453008,
      "cc_tiered_open": 28.967563837129056,
      "cc_tiered_sealed": 28.751778735776956,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 33.33333333333333
     },
     "score": {
      "accuracy": 0.45066666666666666,
      "calibration": 43.78872663887056,
      "cc": 16.56590736182537,
      "cc_by_type": {
       "score": 16.56590736182537
      },
      "cc_tiered": 18.654824979882335,
      "cc_tiered_open": 23.8281765558093,
      "cc_tiered_sealed": 13.48147340395537,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 16.56590736182537
     }
    },
    "raw_on_A": {
     "calibration": 29.76734864606182,
     "capability": 30.080860832336057,
     "intelligence": 30.394373018610295
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 17.177466915718682,
    "Emphasis on Accuracy 60:20:20": 13.36877420821324,
    "Emphasis on Cost 20:20:60": 19.222133829516256,
    "Emphasis on Speed 20:60:20": 20.9104976121348,
    "Intelligence only": 10.032180018050655
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "CodePawl / @anxuanng (founder@codepawl.com)",
   "axes": {
    "calibration": 46.26085085939199,
    "cost": 89.01467380421724,
    "intelligence": 2.2096907014730043,
    "speed": 79.16975860375432
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.43076843750000016,
      "ece_score": 13.846312499999968,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 13.827850749999968,
      "score_before_time_limit": 13.846312499999968,
      "time_limit_failed_share": 0.0013333333333333333
     },
     "noul": {
      "abstention_rate": 0.6746666666666667,
      "brier": 0.28588573402666656,
      "ece": 0.15048053333333336,
      "n": 375,
      "score": 69.71748295111111,
      "score_before_time_limit": 69.90389333333333,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.736,
      "interval90_width": 0.5186666666666669,
      "n": 375,
      "nrps": 0.21227189381999992,
      "score": 55.23721887706489,
      "score_before_time_limit": 55.38491197566667,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.3400149333333333
     }
    },
    "score": 46.26085085939199
   },
   "capability": 24.235270780432494,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.0010142883035394131,
     0.11149985989593862
    ],
    "B": [
     0.0006462160600988027,
     0.07590635392530619
    ],
    "C": [
     0.0007043668774579258,
     0.07743045826106852
    ]
   },
   "cost": {
    "basis": "ESTIMATE (size-class reference; self-hosted open weights, no public tariff). THE SAME CHOICE THE PUBLISHED quyet-1-0-tiny ROW CARRIES FOR THE IDENTICAL mmBERT-small BACKBONE: DeepInfra base-size encoders at USD 0.005 per 1M input, 0 output - deepinfra:thenlper/gte-base, deepinfra:BAAI/bge-base-en-v1.5, deepinfra:intfloat/e5-base-v2 and deepinfra:sentence-transformers/all-mpnet-base-v2 are all (0.005, 0.0) in the frozen 25 Sep snapshot - x this row's own measured usage.input_tokens (the package's packed length, capped at 4,096). Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, verified in the package source (tacet/decoding.py usage_of sets output_tokens 0; the readout is a softmax over option [MASK] markers from one encoder pass and nothing is generated), the published 02 Oct rule for classifier read-outs. PRICING REFERENCE NOTE: jhu-clsp/mmBERT-small still has no explicit market-listing decision (pricing_v15.reference() raises), so an UNLISTED_BASE_MODELS entry would require a published pricing update.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.0026020833333333334
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0023236865267433986
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 3,
    "n_over_limit_answered_ok": 3,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Tacet Sonata (CodePawl, 144M packed encoder on mmBERT-small, option-marker softmax in one pass, in-process via the author's tacet package)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. Weights codepawl/tacet-sonata @ 26f447c35881a80020fe557c95ceb8c61bc3fd12, Apache-2.0, public and ungated, loaded from a local snapshot (no network during the run). Runtime tacet==0.3.0 from PyPI, the author's own install line (`pip install \"tacet>=0.3\"`) in an isolated venv (torch 2.14.1+cpu, transformers 5.19.0). ADAPTER: the author's own PR fstandhartinger/jevbench#197 file tacet_local.py at head 76eb75da, byte-identical (sha256 c63a4fe0...), reviewed before running: it maps answers only (noul P(true) -> yes/no; choice and score probabilities used as returned, score keys '0'..'n-1') and reaches nothing but the Hub. It ran through the UNCHANGED v1.6 driver run_v16.py; the only addition, in a wrapper, registers the file as jevbench.adapters.tacet_local and lists it in IN_PROCESS_MODULES as laya_local is listed - lane verification and the runtime egress guard unchanged. Synthetic smoke gate passed before any sealed item was sent (PREREG-R59.md, receipts/SMOKE-tacet-sonata.json). THE 23 LONG ITEMS ARE TRUNCATED, NOT REFUSED, by the package's own code at its default and documented max_length 4,096 (tacet/packing.py keeps the questions and cuts the state; it refuses only states over 200,000 characters, which no item here reaches): exactly 23 records report input_tokens = 4,096 and they are answered and scored. RUN ON SANDY'S SHARED CPU (12-core Zen 2, AVX2, fp32 as the package runs off-accelerator, 8 threads), load average 6.6-24.2 (median ~14) logged every 60 s: the Speed axis is a DECLARED LOWER BOUND, and the author reports raw p50 0.042 s on an RTX 3060 against this run's 0.265 s. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per call, no retry, no batching, offline, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 716D1957 (bh-submit 07d384eb-fc4e-4536-ba61-888567efdf34)",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "CPU (12-core Zen 2, AVX2; 8 threads)",
   "intelligence": {
    "I_open": 1.6856027951868566,
    "I_sealed": 2.7337786077591524,
    "base": 2.2096907014730043,
    "excess": -3.6215634550105396,
    "gap": -1.0481758125722958,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 5.05680838556057,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 6.5945945945945965,
       "hard": 11.292859389745214,
       "judge": -6.789716545814108,
       "standard": 9.585600669736289
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -65.21739130434783,
       "judge": -70,
       "standard": -66.66666666666667
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 21.489090909090912,
       "hard": -14.746481481481478,
       "judge": 14.17932203389829,
       "standard": -58.12777777777776
      }
     },
     "sealed|choice": {
      "cc": 8.201335823277457,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 16.19244697361616,
       "hard": 4.807692307692292,
       "judge": 5.027932960893876,
       "standard": 15.753171572853812
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -96,
       "hard": -47.57281553398058,
       "judge": -52.63157894736843,
       "standard": -69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 2.2302836879432686,
       "hard": -11.765196078431384,
       "judge": -12.453053892215603,
       "standard": -2.9353299492385965
      }
     }
    }
   },
   "jevbench_score": 0.015686432078748844,
   "key": "tacet-sonata",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "codepawl/tacet-sonata@26f447c35881a80020fe557c95ceb8c61bc3fd12",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.5619834710743802,
    "decisive_rate": 0.3235294117647059,
    "supported": true,
    "valid": 374
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 46.37837260299999,
     "cost": 89.01467380421724,
     "intelligence": 2.2436273530567146,
     "speed": 80.28211114890141
    },
    "board_release": "v1.6.43",
    "capability": 24.310999978028352,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 132,
    "capability_rank": 162,
    "composite_ci95": {
     "A": [
      0.0012369873259327144,
      0.11178055042619336
     ],
     "B": [
      0.0007893069841145717,
      0.07607180414333671
     ],
     "C": [
      0.0008590189763421627,
      0.07762538224041209
     ]
    },
    "jevbench_score": 0.016405006835540154,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 188,
    "rank": 232,
    "ranks": {
     "A": 232,
     "B": 232,
     "C": 232,
     "capability": 234
    },
    "scores": {
     "A": 0.016405006835540154,
     "B": 0.010748475876643845,
     "C": 0.011392365858013995
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.6127247303724289,
     "p50_s_raw": 0.23136236518621445,
     "p95_s_adjusted": 1.5294084161520003,
     "p95_s_raw": 0.6897042080760002
    },
    "views": {
     "Balanced 33:33:33": 0.012868928732057194,
     "Emphasis on Accuracy 60:20:20": 0.007398330998569957,
     "Emphasis on Cost 20:20:60": 0.02046846868350025,
     "Emphasis on Speed 20:60:20": 0.020367269262702412,
     "Intelligence only": 0.004517645714999133
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 273,
   "ranked": true,
   "ranks": {
    "A": 273,
    "B": 273,
    "C": 273,
    "capability": 275
   },
   "repo": "https://huggingface.co/codepawl/tacet-sonata",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.015686432078748844,
    "B": 0.010273103305863243,
    "C": 0.010893355610242255
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.679555581510067,
    "p50_s_raw": 0.2647777907550335,
    "p95_s_adjusted": 1.7815546773374076,
    "p95_s_raw": 0.8157773386687038
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "jhu-clsp/mmBERT-small",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      40.432909045322795,
      49.803976998599616
     ],
     "I": [
      0.8702131419070752,
      4.3653585772798165
     ],
     "capability": [
      20.99119033296382,
      26.57803527930951
     ],
     "composite": {
      "A": [
       0.0010142883035394131,
       0.11149985989593862
      ],
      "B": [
       0.0006462160600988027,
       0.07590635392530619
      ],
      "C": [
       0.0007043668774579258,
       0.07743045826106852
      ]
     },
     "seed": 143
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 46.26085085939199,
     "capability": 24.235270780432494,
     "intelligence": 2.2096907014730043
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2826666666666667,
      "calibration": 13.827850749999968,
      "cc": 8.263311955893602,
      "cc_by_type": {
       "choice": 8.263311955893602
      },
      "cc_tiered": 6.629072104419014,
      "cc_tiered_open": 5.05680838556057,
      "cc_tiered_sealed": 8.201335823277457,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 8.263311955893602
     },
     "noul": {
      "accuracy": 0.18133333333333335,
      "calibration": 69.71748295111111,
      "cc": -63.733333333333334,
      "cc_by_type": {
       "noul": -63.733333333333334
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.304,
      "calibration": 55.23721887706489,
      "cc": -7.684210526315716,
      "cc_by_type": {
       "score": -7.684210526315716
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 45.61976295895299,
     "capability": 24.54098913610771,
     "intelligence": 3.4622153132624316
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.0013333333333332975,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.012298631196612477,
    "Emphasis on Accuracy 60:20:20": 0.007068632431284845,
    "Emphasis on Cost 20:20:60": 0.019574566887244516,
    "Emphasis on Speed 20:60:20": 0.019465551712092424,
    "Intelligence only": 0.004315731879770439
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "tdelab",
   "axes": {
    "calibration": 56.064346266900806,
    "cost": null,
    "intelligence": 5.640752949655241,
    "speed": 93.94749802780368
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.23272817754556263,
      "ece_score": 53.45436449088747,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 53.45436449088747,
      "score_before_time_limit": 53.45436449088747,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.5386666666666666,
      "brier": 0.3053899527599847,
      "ece": 0.22745442238985986,
      "n": 375,
      "score": 54.509115522028026,
      "score_before_time_limit": 54.509115522028026,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.768,
      "interval90_width": 0.5253333333333334,
      "n": 375,
      "nrps": 0.20232630026253115,
      "score": 60.229558787786935,
      "score_before_time_limit": 60.229558787786935,
      "time_limit_failed_share": 0,
      "top_ece": 0.29654126199086506
     }
    },
    "score": 56.064346266900806
   },
   "capability": 30.852549608278025,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED. The frozen pricing_v15 module (sha256 3948e3f1..., and the later re-freezes, which only add or re-price hosted-API launch-list keys) has no market-listing decision for the declared base answerdotai/ModernBERT-large (the module lists it in UNLISTED_BASE_MODELS; the card declares no base_model); no price is invented here. Axes only: Intelligence, Calibration and Speed are computed; Cost, composite and rank are void. Tokens: the server reports no token field, so input is counted OFFLINE with the model's own tokenizer on the pod (the counter of the earlier measurement, 672,464 over the full draw, including the 1,024-token truncation) and output is 0 under the 9 Oct 2026 rule (encoder classifier, nothing generated).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "TDE general large v0.1 (tdelab, ModernBERT-large encoder)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-16",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Lium GPU pod (measurement receipts on file), 1x NVIDIA H100 80 GB HBM3 (sm_90, driver 580.178.04, United States node, pre-registration Amendments 1-2) - the card most of these systems were first evaluated on, but a different pod, stack and pin set from the earlier runs, so Speed is as measured on this card and probabilities may differ from the earlier runs in the last digits. Python packaging: uv venv from a hashed lock at the old pins, dependencies resolved as of the old build time where the old freeze is lost (declared deviation, UNVERIFIED where noted). Isolation: author server started inside a network namespace with loopback only, the cut proven before and after the sealed input was staged, synthetic smoke gate passed in between; runner unchanged (run_v15.py --resume --resume-keep-failures); one use of input-selfhosted-1500 (under the pre-registration). Published Speed uses the standing x2 + 0.15 s self-hosted adjustment. Author's tde-serve, one forward pass.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 7.694898428398389,
    "I_sealed": 3.586607470912093,
    "base": 5.640752949655241,
    "excess": -5.731235812869415,
    "gap": 4.108290957486296,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 23.084695285195167,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 33.837837837837846,
       "hard": 22.61717521233092,
       "judge": 9.030982201713911,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -42.85714285714286,
       "hard": -47.82608695652174,
       "judge": -40,
       "standard": -22.22222222222222
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 15.912047352387827,
       "hard": -26.723417420160132,
       "judge": -8.499775844886592,
       "standard": -0.3084053475133963
      }
     },
     "sealed|choice": {
      "cc": 10.75982241273628,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 6.607369758576881,
       "hard": 6.325893373424297,
       "judge": 13.61410381978454,
       "standard": 17.422484707867554
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -44.827586206896555,
       "hard": -61.81818181818181,
       "judge": -44.61538461538461,
       "standard": -37.31343283582089
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 6.072994815539278,
       "hard": -17.474036712402707,
       "judge": -17.468055468937884,
       "standard": 3.607221118509185
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "tde-general-large",
   "last_measured_on": "2026-10-16",
   "licence": "Apache-2.0",
   "listing": "listed",
   "measured_in": "v1.6.26",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "tdelab/tde-general-large-v0.1@2d697981fc217d50c27d042079549c555bc08e27",
   "not_ranked_because": "unpriced: answerdotai/ModernBERT-large is in the frozen module's UNLISTED_BASE_MODELS (no eligible base-model price); one listing decision would make the row rankable.",
   "noul_decisive": {
    "acc_among_decisive": 0.5722543352601156,
    "decisive": 173,
    "decisive_rate": 0.4613333333333333,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 56.064346266900806,
     "cost": null,
     "intelligence": 5.640752949655241,
     "speed": 93.9202043701742
    },
    "board_release": "v1.6.43",
    "capability": 30.852549608278025,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.26",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.19189397068694233,
     "p50_s_raw": 0.02094698534347117,
     "p95_s_adjusted": 0.21130911619402468,
     "p95_s_raw": 0.03065455809701234
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/tdelab/tde-general-large-v0.1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.1899578004144132,
    "p50_s_raw": 0.019978900207206607,
    "p95_s_adjusted": 0.2121255909092724,
    "p95_s_raw": 0.031062795454636215
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.3181818181818182,
       "cc": -44.64889836187164,
       "cc_by_type": {
        "choice": 0,
        "noul": 0,
        "score": -133.94669508561492
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.13861386138613863,
       "cc": -47.882357699574925,
       "cc_by_type": {
        "choice": -14.528101802757151,
        "noul": -92,
        "score": -37.11897129596764
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.32673267326732675,
       "cc": -11.452348280944049,
       "cc_by_type": {
        "choice": 5.691056910569103,
        "noul": -40.74074074074075,
        "score": 0.6926389873395
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.5625,
       "cc": 27.37802379454176,
       "cc_by_type": {
        "choice": 100,
        "noul": -19.999999999999996,
        "score": 2.13407138362528
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 27.37802379454176
      },
      "jaggedness": {
       "accuracy": 0.24444444444444444,
       "cc": -18.086957713428244,
       "cc_by_type": {
        "choice": -5.389797882579387,
        "noul": -56.75675675675676,
        "score": 7.885681499051412
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.41214057507987223,
       "cc": 10.23603012074738,
       "cc_by_type": {
        "choice": 24.716034163951754,
        "noul": -13.4020618556701,
        "score": 19.394118053960486
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 10.23603012074738
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": -4.088753649103704,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": 100,
        "score": -109.04045449569821
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.2962962962962963,
       "cc": -35.789280202050165,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -100,
        "score": -33.141036482439155
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.23272727272727273,
       "cc": -33.64829691573089,
       "cc_by_type": {
        "choice": 4.1408542549722975,
        "noul": -75,
        "score": -30.085745002164966
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.3372093023255814,
       "cc": -19.075778871774677,
       "cc_by_type": {
        "choice": 20.000000000000004,
        "noul": -60,
        "score": -17.227336615324027
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.4375,
       "cc": 28.095872170439414,
       "cc_by_type": {
        "choice": 28.095872170439414
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 28.095872170439414
      },
      "perturb_pair": {
       "accuracy": 0.390625,
       "cc": 8.380270527016437,
       "cc_by_type": {
        "choice": 21.921921921921925,
        "noul": 0,
        "score": 3.2188896591273886
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 8.380270527016437
      },
      "usecase": {
       "accuracy": 0.2636363636363636,
       "cc": -18.700683659499784,
       "cc_by_type": {
        "choice": 15.44117647058827,
        "noul": -58.33333333333333,
        "score": -13.209894115754285
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -31.210698543050498,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": -100,
        "score": -55.5368575339134
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0,
       "cc": -63.15789473684211,
       "cc_by_type": {
        "choice": -26.315789473684216,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.3076923076923077,
       "cc": -10.355750352231984,
       "cc_by_type": {
        "choice": 17.050691244239633,
        "noul": -33.333333333333336,
        "score": -14.784608967602253
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.34375,
       "cc": 0.3767855970192109,
       "cc_by_type": {
        "choice": 12.389380530973451,
        "noul": -25,
        "score": 13.740976260084182
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0.3767855970192109
      },
      "el": {
       "accuracy": 0.2,
       "cc": -26.806881135846425,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100,
        "score": 3.3782392740249634
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.3214596003475239,
       "cc": -10.566948030932052,
       "cc_by_type": {
        "choice": 14.548272537557459,
        "noul": -41.72661870503597,
        "score": -4.522497925317648
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.4838709677419355,
       "cc": -10.39293718864508,
       "cc_by_type": {
        "choice": 41.27243066884176,
        "noul": 0,
        "score": -72.451242234777
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -37.651823606267435,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": -47.381700326999024
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.14285714285714285,
       "cc": -47.5054349799874,
       "cc_by_type": {
        "choice": -27.05882352941176,
        "noul": -100,
        "score": -15.457481410550432
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.09523809523809523,
       "cc": -79.1298022954612,
       "cc_by_type": {
        "choice": -34.07821229050279,
        "noul": -100,
        "score": -103.3111945958808
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.4,
       "cc": -2.400158538865593,
       "cc_by_type": {
        "choice": 15.822291504286824,
        "noul": -33.333333333333336,
        "score": 10.310566212449734
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.16,
       "cc": -30.317930523064486,
       "cc_by_type": {
        "choice": -14.376489277204131,
        "noul": -80,
        "score": 3.42269770801068
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.1,
       "cc": -60.65000059630072,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": -48.61666845556882
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.35714285714285715,
       "cc": -0.3190550228787809,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -33.333333333333336,
        "score": -0.9571650686363453
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.3,
       "cc": -32.239066920314436,
       "cc_by_type": {
        "choice": -3.8961038961038925,
        "noul": -33.333333333333336,
        "score": -59.48776353150607
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 25,
       "cc_by_type": {
        "choice": 100,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 25
      },
      "pl": {
       "accuracy": 0.2564102564102564,
       "cc": -31.55678173364039,
       "cc_by_type": {
        "choice": 31.443080829454082,
        "noul": -80,
        "score": -46.11342603037525
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.2857142857142857,
       "cc": -19.237636121315095,
       "cc_by_type": {
        "choice": -16.676487919858577,
        "noul": -19.999999999999996,
        "score": -21.036420444086712
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0,
       "cc": -45.1384804957936,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": -100,
        "score": -7.755867019295692
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.23076923076923078,
       "cc": -32.401275767040424,
       "cc_by_type": {
        "choice": -8.695652173913041,
        "noul": -100,
        "score": 11.491824872791767
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.14285714285714285,
       "cc": -52.86352895100111,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100,
        "score": -61.81639330461621
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.4,
       "cc": 11.717145125295701,
       "cc_by_type": {
        "choice": -1.694915254237285,
        "noul": 33.33333333333333,
        "score": 3.5130172967910656
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 11.717145125295701
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.3293333333333333,
       "cc": 13.373772938632586,
       "cc_by_type": {
        "choice": 13.373772938632586
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 13.373772938632586
      },
      "noul": {
       "accuracy": 0.264,
       "cc": -47.199999999999996,
       "cc_by_type": {
        "noul": -47.199999999999996
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.31466666666666665,
       "cc": -9.875527857288091,
       "cc_by_type": {
        "score": -9.875527857288091
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -35.61446330550255,
       "cc_by_type": {
        "choice": 36,
        "noul": -75,
        "score": -67.84338991650765
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.25,
       "cc": -21.70823191758269,
       "cc_by_type": {
        "choice": -11.801242236024848,
        "noul": -33.333333333333336,
        "score": -19.99012018338988
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.2857142857142857,
       "cc": -27.66666238557114,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": -100,
        "score": -17.959336750209353
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -43.7112086980384,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": -49.262281064875424
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.29411764705882354,
       "cc": -34.37213530485037,
       "cc_by_type": {
        "choice": -1.4492753623188428,
        "noul": -100,
        "score": -1.6671305522322655
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.2631578947368421,
       "cc": -109.14827947127996,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -100,
        "score": -239.39452394843107
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.26666666666666666,
       "cc": -19.614304130752316,
       "cc_by_type": {
        "choice": 31.372549019607845,
        "noul": -71.42857142857143,
        "score": -18.786889983293364
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -20.713998010360054,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -50,
        "score": -28.51830065128922
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.06666666666666667,
       "cc": -35.32681576476467,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -100,
        "score": -1.8137806276273372
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": -4.430408096434276,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": -33.333333333333336,
        "score": 9.977869215336721
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.24,
       "cc": -21.949397594541157,
       "cc_by_type": {
        "choice": 18.604651162790695,
        "noul": -55.55555555555556,
        "score": -28.897288390858613
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.1,
       "cc": -30.553913197904084,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -33.333333333333336,
        "score": -29.29614819586279
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.1111111111111111,
       "cc": -38.5995859651619,
       "cc_by_type": {
        "choice": -15.163147792706338,
        "noul": -60,
        "score": -40.63561010277936
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.45161290322580644,
       "cc": 15.210953969333254,
       "cc_by_type": {
        "choice": 32.20779220779223,
        "noul": -6.060606060606055,
        "score": 19.48567576081358
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 15.210953969333254
      },
      "moderation_trust_safety": {
       "accuracy": 0.3333333333333333,
       "cc": -22.93724248098899,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100,
        "score": -11.230153546613808
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.2777777777777778,
       "cc": -28.345864082899254,
       "cc_by_type": {
        "choice": 16.434540389972145,
        "noul": -100,
        "score": -1.4721326386698985
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.1111111111111111,
       "cc": -57.21913205390808,
       "cc_by_type": {
        "choice": -18.226600985221676,
        "noul": -100,
        "score": -53.430795176502556
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.2608695652173913,
       "cc": -28.737735566525654,
       "cc_by_type": {
        "choice": -15.942028985507239,
        "noul": -66.66666666666667,
        "score": -3.6045110474030473
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.2916666666666667,
       "cc": -24.20298232911786,
       "cc_by_type": {
        "choice": 25.26690391459075,
        "noul": -100,
        "score": 2.124149098055661
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      50.262207692557865,
      60.42640133753659
     ],
     "I": [
      3.74912535565067,
      7.628551559966515
     ],
     "capability": [
      27.807284767450415,
      33.53931156979206
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 75
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 56.064346266900806,
     "capability": 30.852549608278025,
     "intelligence": 5.640752949655241
    },
    "per_type": {
     "choice": {
      "accuracy": 0.3293333333333333,
      "calibration": 53.45436449088747,
      "cc": 13.373772938632586,
      "cc_by_type": {
       "choice": 13.373772938632586
      },
      "cc_tiered": 16.922258848965726,
      "cc_tiered_open": 23.084695285195167,
      "cc_tiered_sealed": 10.75982241273628,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.373772938632586
     },
     "noul": {
      "accuracy": 0.264,
      "calibration": 54.509115522028026,
      "cc": -47.199999999999996,
      "cc_by_type": {
       "noul": -47.199999999999996
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.31466666666666665,
      "calibration": 60.229558787786935,
      "cc": -9.875527857288091,
      "cc_by_type": {
       "score": -9.875527857288091
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 53.12005786809952,
     "capability": 29.312187813772837,
     "intelligence": 5.504317759446154
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "matthewagi (Hugging Face)",
   "axes": {
    "calibration": 88.71404111864382,
    "cost": 71.33135335381719,
    "intelligence": 44.07459281825939,
    "speed": 90.72909335832355
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06361399351573381,
      "ece_score": 87.27720129685324,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 87.27720129685324,
      "score_before_time_limit": 87.27720129685324,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3726541554959786,
      "brier": 0.11603929170212504,
      "ece": 0.07014905306792772,
      "n": 373,
      "score": 85.97018938641445,
      "score_before_time_limit": 85.97018938641445,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.967741935483871,
      "interval90_width": 0.45833333333333365,
      "n": 372,
      "nrps": 0.08941220418134077,
      "score": 92.89473267266376,
      "score_before_time_limit": 92.89473267266376,
      "time_limit_failed_share": 0,
      "top_ece": 0.026346571182691923
     }
    },
    "score": 88.71404111864382
   },
   "capability": 66.3943169684516,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 842,725 input + 0 output tokens over 1,485 decisions = 567.491582492 tokens/decision. GPU-time cost USD 0.00902862374655256/1,000; old list-price cost USD 0.0851237373737374/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00902862374655256/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0842725
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009028623746552559
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Tern-35B-A3B (matthewagi, Qwen3.6 35B-A3B MoE decision model, author server, self-hosted)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (driver 580.178.04), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.525/h. The author's own `python3 -m tern serve` from the pinned commit on 127.0.0.1:8000 (vLLM backend on 127.0.0.1:8101) inside a network-none Docker container with the cut proven from inside (loopback only, no default route, DNS and TCP fail), read-only root, tmpfs scratch, read-only weights, harness and task input, empty credential placeholders, HF offline flags, one request at a time (no concurrency, no batching endpoint). Timed over loopback on the VM. One disclosed deviation: a read-write compile-cache bind shared between the synthetic smoke container and the scored container, so the second cold start fits the time limit; it carries compiler artifacts only and leaves content on the pod disk, which is deleted. Measured 22 Oct 2026 (VM reserved 06:29Z, removed 06:48Z) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720..., checked locally, on the node and inside the container); determinism 50/50 bit-identical; node shredded and VM destroyed afterwards (teardown confirmed).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 50.428471552943506,
    "I_sealed": 37.720714083575274,
    "base": 44.07459281825939,
    "excess": 2.8682306990125213,
    "gap": 12.707757469368232,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 76.60705387746165,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 77.35136835482857,
       "judge": 72.31377719182598,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 28.017253278122844,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 39.13043478260869,
       "judge": 10.000000000000009,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 46.661107503246036,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 49.43405451873939,
       "hard": 48.41853406073319,
       "judge": 43.25118143648162,
       "standard": 46.874669980671655
      }
     },
     "sealed|choice": {
      "cc": 63.47155961797452,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 84.75222363405337,
       "hard": 57.32624031455995,
       "judge": 58.570029382957905,
       "standard": 72.47416156928918
      }
     },
     "sealed|noul": {
      "cc": 13.087561949116248,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 34.48275862068966,
       "hard": 7.272727272727275,
       "judge": 1.538461538461533,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 36.603020683635066,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 58.76270966326589,
       "hard": 26.38182260397771,
       "judge": 39.45655585509179,
       "standard": 41.68526959594928
      }
     }
    }
   },
   "jevbench_score": 52.678174120800755,
   "key": "tern-35b-a3b",
   "last_measured_on": "2026-10-22",
   "licence": "Apache-2.0 (HF card metadata and model LICENSE; the Qwen base licence is retained as BASE_MODEL_LICENSE; code repo Apache-2.0), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "matthewagi/Tern-35B-A3B@ef783c939186c1363f1de46d63564e78bb22f1c5",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9572649572649573,
    "decisive": 234,
    "decisive_rate": 0.6273458445040214,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 119,
   "ranked": true,
   "ranks": {
    "A": 119,
    "B": 121,
    "C": 126,
    "capability": 123
   },
   "repo": "https://huggingface.co/matthewagi/Tern-35B-A3B",
   "scores": {
    "A": 52.678174120800755,
    "B": 47.55914405570292,
    "C": 36.58206536166719
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2874846070000331,
    "p50_s_raw": 0.06874230350001653,
    "p95_s_adjusted": 0.2940871632998323,
    "p95_s_raw": 0.07204358164991617
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "The author's own `tern` package (Matthew-agi/Tern at d154fa085d92eb2170681cd90817e18c735553ff, tree 529c3f40c3cc966b15b3e964a5c79ea9bde6c894, Apache-2.0, 66 files, every git blob oid verified; no tag exists on that repo; the README-tested commit 9723b745 differs only in README/docs), unmodified, installed from the pinned tree into our own image and started as `python3 -m tern serve --profile rtx-pro-6000 --model <pinned verified snapshot> --cache-dir /cache/tern --gpu 0 --host 127.0.0.1 --port 8000 --backend-port 8101` (the author's Docker command plus our local --model). Author profile kept unchanged: max_model_len 32,768, 32,768 batch tokens, 256 seqs, CUDA-graph capture 16,384, BF16, Triton recurrent kernels, head batch 16/16,384, CUBLAS_WORKSPACE_CONFIG=:4096:8, CLEF_SUBMISSION_MODE=single, all 40 layers and 8 experts, no result cache; the h100, h100-bf16 and a100 profiles and the HF runtime/serve.py were not used. Static source review PASS WITH FINDINGS, no blockers (no non-loopback network, no telemetry, no item logging, no pickle/trust_remote_code, no rule keyed to item text or benchmark names). DISCLOSED CODE PATTERNS, cleared by the benchmark owner (21 Oct 2026; author code on a rented Verda pod only, weights pre-staged, network none): (1) the author's default start downloads the 33 model files from the Hub (src/tern/artifacts.py:36-56, called from launch.py:105) - never reached, because --model points to a pre-staged local snapshot (artifacts.py:37-38 returns before any Hub import), with HF offline flags, empty credentials and a network-none container; (2) a second snapshot_download in an unused research helper (kev_vllm/clef_schema.py:501-507, no caller); (3) an exec-based rewrite of one vLLM function (kev_vllm/expert_act_quant.py:75), off in the RTX BF16 profile; (4) vLLM monkey-patches behind the author's own sha256 check of 16 vLLM source files (fail-closed); (5) subprocess calls (nvidia-smi and the server's own two child processes, list argv, no shell; src/tern/launch.py:25,33,122) inside the network-none container. Weights: matthewagi/Tern-35B-A3B at ef783c939186c1363f1de46d63564e78bb22f1c5, exactly the author's 33 inference files (69,778,082,886 bytes), fetched by us from the commit-pinned Hub URLs and sha256-verified file by file on the host, then re-hashed by the author's own verify() at every container start. Image: our own build on the pod (network on, before any sealed byte existed) from the author's digest-pinned base nvidia/cuda:12.9.1-devel-ubuntu24.04 plus the author's 196 pins re-expressed with published sha256 hashes (--require-hashes) and the pinned source tree; torch 2.13.0+cu129, vllm 0.30.0+cu129, transformers 5.19.0, triton 3.7.1, flashinfer 0.6.18.post1; image id sha256:c097beb417b5ea0d34ea5a5c144a7159d42e79dad8e9067847cce00f9e5e7ae1. FIRST MEASUREMENT (found by our own intake: Decision Index 0.4.1 entry 56, Hugging Face decision-index tag). AUTHOR-DEFAULT CONTEXT CAP: the author's server answers HTTP 422 natively above its 32,768-token limit (whole state plus all questions and options, strict >, no truncation; serve_clef_api.py:59-60,149-150); no status proxy was used. The 15 items of 76k-81k tokens (76,424-80,587) were refused exactly as pre-registered by id and position and are scored wrong, like the other self-hosted rows on this draw; the largest answered item is 1,886 tokens and none is within 64 tokens of the limit. SERIAL ONLY: the author states that answers shift with server load and batching (the Decision Index lab found 98.39% agreement with the submitted rows on 827 rows against its 98.5% bar, which it attributes to 128 concurrent requests in the submitted run versus one at a time in the check; listed under a maintainer decision); we sent strictly one request at a time, and determinism over the first 50 positions was bit-identical (50/50, 0 argmax flips). The probabilities need not equal the author's or the DI lab's (author: cross-GPU and cross-batch differences are unproven equivalent). TRAINING DATA, the author's own statement (TRAINING_DISCLOSURE.md, unverified): full-depth Qwen3.6-35B-A3B backbone with merged adapters (checkpoint 1674, about 500M tokens) and a jointly trained decision head, trained on 52 source families that include the teacher-derived jev_distill and open_jev data and train splits of public benchmarks (for example BANKING77, CLINC, HoVer, GSM8K, When2Call); 150 Decision Index-derived BBH, MMLU-Pro and When2Call requests were observed during development; the author's overlap audit lists 3,238 candidate requests with shared text among the 140,178 public Decision Index requests, 1,392 of them in index benchmarks (not proven identical decisions), and states this is not an uncontaminated held-out claim. Our own check of the exact published text hashes against our public 231 items found 0 hits; it cannot see teacher-derived knowledge or unpublished training text. The author flags 8,678 ANLI and 4,304 ContractNLI training records tagged non-commercial in intermediate dataset metadata (author: not a conclusion about the weights); the model is licensed Apache-2.0 and not marked non-commercial. SELF-REPORTED, unverified: public Decision Index score 59.01 on RTX PRO 6000, warmed serial latency median 59.48 ms on the author's card. 10 s RULE: max measured decision latency 0.092 s over all 1,500 rows; 0 decisions over 10 s.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 74.08010601583455,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 55.573651380836964
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 74.08010601583455
      },
      "calibration_gen": {
       "accuracy": 0.45544554455445546,
       "cc": 14.58961955883486,
       "cc_by_type": {
        "choice": 36.37327677624603,
        "noul": 4.0000000000000036,
        "score": 3.3955819002585486
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 14.58961955883486
      },
      "composite_gen": {
       "accuracy": 0.7128712871287128,
       "cc": 42.81849470657943,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -3.703703703703709,
        "score": 48.419350425068
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 42.81849470657943
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.10507525772684,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 55.31522577318052
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.10507525772684
      },
      "jaggedness": {
       "accuracy": 0.5277777777777778,
       "cc": 22.74275699647896,
       "cc_by_type": {
        "choice": 43.69586140519731,
        "noul": -13.513513513513509,
        "score": 38.04592309775307
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 22.74275699647896
      },
      "llm_router": {
       "accuracy": 0.7667731629392971,
       "cc": 61.80122015484502,
       "cc_by_type": {
        "choice": 71.47133926212908,
        "noul": 58.762886597938135,
        "score": 55.169434604467824
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 61.80122015484502
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": -9.903319954898492,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": -75,
        "score": -5.225423782221261
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6545454545454545,
       "cc": 37.785915212351604,
       "cc_by_type": {
        "choice": 69.67720899902184,
        "noul": -2.777777777777779,
        "score": 46.45831441581074
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 37.785915212351604
      },
      "negation_pair_gen": {
       "accuracy": 0.7093023255813954,
       "cc": 32.55812757855488,
       "cc_by_type": {
        "choice": 96.66666666666667,
        "noul": -19.999999999999996,
        "score": 21.007716068997983
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 32.55812757855488
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 78.40789816963463,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100,
        "score": 59.24771853292792
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 78.40789816963463
      },
      "usecase": {
       "accuracy": 0.65,
       "cc": 42.58590768435422,
       "cc_by_type": {
        "choice": 69.36274509803921,
        "noul": 16.666666666666675,
        "score": 41.728311288356764
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 42.58590768435422
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.625,
       "cc": 18.21433405809204,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": -57.26175973048579
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 18.21433405809204
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 60.8667161853777,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 49.26681522279978
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 60.8667161853777
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 75.73588042373903,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 76.10144658095163
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 75.73588042373903
      },
      "el": {
       "accuracy": 0.4,
       "cc": 21.37421355089219,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 97.82841346459463
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.37421355089219
      },
      "en": {
       "accuracy": 0.6759339704604691,
       "cc": 43.172496055235456,
       "cc_by_type": {
        "choice": 66.82979216753516,
        "noul": 23.741007194244613,
        "score": 38.9466888039266
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 43.172496055235456
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 66.70348822602944,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 9.898392899948005
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 66.70348822602944
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 42.8880393396527,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 93.69144042332968
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 42.8880393396527
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 85.8572915694662,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 78.74834529663389
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 85.8572915694662
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 32.83579577600802,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -25,
        "score": 40.26716386433691
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 32.83579577600802
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.7333333333333333,
       "cc": 55.92058462532583,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -18.208628041403642
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 55.92058462532583
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 13.024478812283887,
       "cc_by_type": {
        "choice": 28.514694201747414,
        "noul": 0,
        "score": 10.558742235104246
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 13.024478812283887
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -11.465428872083232,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 10.048157828194759
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 70.13123827831198,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 93.7270481682693
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 70.13123827831198
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -7.542437762022018,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": -22.627313286066062
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 35.69444255134223,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": -19.999999999999996,
        "score": 42.318198580814645
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 35.69444255134223
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 58.23344183481286,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 60.00000000000001,
        "score": 46.521185846218174
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 58.23344183481286
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": 3.902465116747834,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60,
        "score": 73.83505492471159
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 3.902465116747834
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 14.384514212851292,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 0,
        "score": -35.10732692666352
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 14.384514212851292
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 72.65884973589452,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 82.4926782399416
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 72.65884973589452
      },
      "zh": {
       "accuracy": 0.7,
       "cc": 25.3227144997153,
       "cc_by_type": {
        "choice": 100,
        "noul": -33.333333333333336,
        "score": 9.301476832479239
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 25.3227144997153
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.748,
       "cc": 67.45058267475459,
       "cc_by_type": {
        "choice": 67.45058267475459
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 67.45058267475459
      },
      "noul": {
       "accuracy": 0.5973333333333334,
       "cc": 19.466666666666676,
       "cc_by_type": {
        "noul": 19.466666666666676
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 19.466666666666676
      },
      "score": {
       "accuracy": 0.5946666666666667,
       "cc": 39.24988739460201,
       "cc_by_type": {
        "score": 39.24988739460201
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 39.24988739460201
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -8.428579510134812,
       "cc_by_type": {
        "choice": 52,
        "noul": -75,
        "score": -2.2857385304044353
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 26.598713535860508,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 11.111111111111116,
        "score": 5.952110241812014
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 26.598713535860508
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 38.90421156137979,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50,
        "score": -17.027202714234612
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 38.90421156137979
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -33.96164054317912,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -31.709483033046126
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 18.741946383761825,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 33.03743335418403
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 18.741946383761825
      },
      "feature_extraction": {
       "accuracy": 0.5789473684210527,
       "cc": -12.090688196852398,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 33.33333333333333,
        "score": -106.71231616288424
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 37.27941422899224,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 55.535721678573346
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 37.27941422899224
      },
      "gaming": {
       "accuracy": 0.5454545454545454,
       "cc": 4.196787496841249,
       "cc_by_type": {
        "choice": 68.6411149825784,
        "noul": -100,
        "score": 43.949247507945344
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 4.196787496841249
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 28.96044993083234,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": -19.999999999999996,
        "score": 48.54801645916369
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 28.96044993083234
      },
      "knowledge_graphs": {
       "accuracy": 0.5909090909090909,
       "cc": 43.26818788361002,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 47.863093272528864
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 43.26818788361002
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 50.09458538469488,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 28.578329797495496
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 50.09458538469488
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -28.18153018272051,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100,
        "score": -20.028461515903473
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 33.99841601057772,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -19.999999999999996,
        "score": 33.511562811003785
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 33.99841601057772
      },
      "model_routing": {
       "accuracy": 0.7880184331797235,
       "cc": 63.33652101229035,
       "cc_by_type": {
        "choice": 81.2987012987013,
        "noul": 66.66666666666667,
        "score": 42.04419507150307
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 63.33652101229035
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 3.642456085476842,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -100,
        "score": 56.99262737351306
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 3.642456085476842
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 56.69479476307428,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 53.46414287789509
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 56.69479476307428
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 17.422823881920966,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": -50,
        "score": 31.82512189206831
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 17.422823881920966
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 3.548736057529934,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -66.66666666666667,
        "score": 35.28388933201009
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 3.548736057529934
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 77.87106760875567,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 83.61320282626701
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 77.87106760875567
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.9737930322135,
      88.87866305138917
     ],
     "I": [
      38.25178041426043,
      48.41158630707976
     ],
     "capability": [
      61.60504955133157,
      67.74481906925115
     ],
     "composite": null,
     "seed": 79
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.71404111864382,
     "capability": 66.3943169684516,
     "intelligence": 44.07459281825939
    },
    "per_type": {
     "choice": {
      "accuracy": 0.748,
      "calibration": 87.27720129685324,
      "cc": 67.45058267475459,
      "cc_by_type": {
       "choice": 67.45058267475459
      },
      "cc_tiered": 70.03930674771809,
      "cc_tiered_open": 76.60705387746165,
      "cc_tiered_sealed": 63.47155961797452,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 67.45058267475459
     },
     "noul": {
      "accuracy": 0.5973333333333334,
      "calibration": 85.97018938641445,
      "cc": 19.466666666666676,
      "cc_by_type": {
       "noul": 19.466666666666676
      },
      "cc_tiered": 20.552407613619547,
      "cc_tiered_open": 28.017253278122844,
      "cc_tiered_sealed": 13.087561949116248,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 19.466666666666676
     },
     "score": {
      "accuracy": 0.5946666666666667,
      "calibration": 92.89473267266376,
      "cc": 39.24988739460201,
      "cc_by_type": {
       "score": 39.24988739460201
      },
      "cc_tiered": 41.63206409344055,
      "cc_tiered_open": 46.661107503246036,
      "cc_tiered_sealed": 36.603020683635066,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 39.24988739460201
     }
    },
    "raw_on_A": {
     "calibration": 83.4083275232536,
     "capability": 65.09795629748626,
     "intelligence": 46.78758507171892
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 48.83925694266709,
    "Emphasis on Accuracy 60:20:20": 41.72753627264297,
    "Emphasis on Cost 20:20:60": 51.27687075936904,
    "Emphasis on Speed 20:60:20": 55.682277211700054,
    "Intelligence only": 34.247187985271324
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 82.93469131589123,
    "cost": 56.36034169226667,
    "intelligence": 60.54320091058725,
    "speed": 91.34133469499314
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07394265024295099,
      "ece_score": 85.2114699514098,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 85.2114699514098,
      "score_before_time_limit": 85.2114699514098,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.035040431266846306,
      "brier": 0.1393319820198383,
      "ece": 0.13485522993675053,
      "n": 371,
      "score": 73.02895401264989,
      "score_before_time_limit": 73.02895401264989,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.964769647696477,
      "interval90_width": 0.33220415537488723,
      "n": 369,
      "nrps": 0.06922052747100878,
      "score": 90.56364998361398,
      "score_before_time_limit": 90.56364998361398,
      "time_limit_failed_share": 0,
      "top_ece": 0.05975323642835579
     }
    },
    "score": 82.93469131589123
   },
   "capability": 71.73894611323924,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     67.16941178445903,
     71.41294357282025
    ],
    "B": [
     64.31810039127373,
     69.97094141798148
    ],
    "C": [
     56.54608963400217,
     71.41294357282025
    ]
   },
   "cost": {
    "basis": "v1.5.8 fast-lane delivery row (combined v1.5.8 preview, not yet published) cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): ESTIMATE (v1.5-M2 base-model reference; no public tariff, customer asked for the base-model reference): OpenRouter google/gemma-3-12b-it list price $0.05/M input, $0.15/M output (gemma-4-12b-it is not listed on OpenRouter, read 2026-10-04; same reference as the Cygnet and Jev-Omni Gemma-4-12B rows) x the shim's own measured usage.prompt_tokens/completion_tokens; self-hosted on 1x H100 80GB (Lium), vLLM 0.30.0 BF16; estimated, not charged",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02848756157635468
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "torchcast-decision-12b (Torchcast AI, Gemma-4-12B fine-tune, option-letter logprob readout)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, weights fetched in a separate model-only container, scored run inside docker --network none (loopback only, no secrets, label-free inputs); pod destroyed afterwards; paid public fast-lane evaluation, 4 Oct 2026 (paid evaluation)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 59.97536524790964,
    "I_sealed": 61.11103657326486,
    "base": 60.54320091058725,
    "excess": -3.7090589677934624,
    "gap": -1.1356713253552186,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 71.98506861847196,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 68.35860250494396,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 56.49758454106281,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 50,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 51.44344258419417,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.22709032709656,
       "hard": 57.48378737898936,
       "judge": 45.79568230097426,
       "standard": 44.44256954798248
      }
     },
     "sealed|choice": {
      "cc": 68.40340626918382,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 60.80316742081447,
       "judge": 72.06703910614526,
       "standard": 72.28722749107035
      }
     },
     "sealed|noul": {
      "cc": 62.37236788127848,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 72,
       "hard": 45.63106796116505,
       "judge": 65.78947368421053,
       "standard": 85.91549295774648
      }
     },
     "sealed|score": {
      "cc": 52.557335569332295,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 69.05633413996496,
       "hard": 44.66535408470005,
       "judge": 55.981937871697106,
       "standard": 54.95489579973323
      }
     }
    }
   },
   "jevbench_score": 69.84980331350847,
   "key": "torchcast-decision-12b",
   "last_measured_on": "2026-10-04",
   "licence": "cc-by-nc-4.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8575418994413407,
    "decisive_rate": 0.9649595687331537,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.93469131589123,
     "cost": 56.36034169226667,
     "intelligence": 60.54320091058725,
     "speed": 91.74573681567192
    },
    "board_release": "v1.6.43",
    "capability": 71.73894611323924,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 20,
    "capability_rank": 37,
    "composite_ci95": {
     "A": [
      67.22388654747238,
      71.4745219700647
     ],
     "B": [
      64.35805126511531,
      70.01822593897899
     ],
     "C": [
      56.59214410663297,
      71.4745219700647
     ]
    },
    "jevbench_score": 69.9087143535159,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 11,
    "rank": 27,
    "ranks": {
     "A": 27,
     "B": 32,
     "C": 25,
     "capability": 64
    },
    "scores": {
     "A": 69.9087143535159,
     "B": 67.81076568849952,
     "C": 69.9087143535159
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2243030747398734,
     "p50_s_raw": 0.037151537369936705,
     "p95_s_adjusted": 0.29825730649754406,
     "p95_s_raw": 0.07412865324877203
    },
    "views": {
     "Balanced 33:33:33": 66.43077398924878,
     "Emphasis on Accuracy 60:20:20": 63.94347960496841,
     "Emphasis on Cost 20:20:60": 61.999556455891316,
     "Emphasis on Speed 20:60:20": 74.67237011934479,
     "Intelligence only": 60.54320091058725
    }
   },
   "provenance": {
    "carried_metadata_from": "author submission",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 80,
   "ranked": true,
   "ranks": {
    "A": 80,
    "B": 72,
    "C": 61,
    "capability": 81
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-12b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.84980331350847,
    "B": 67.76641469720529,
    "C": 69.84980331350847
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23964857459068298,
    "p50_s_raw": 0.04482428729534149,
    "p95_s_adjusted": 0.30640205964446066,
    "p95_s_raw": 0.07820102982223033
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.04707649811247,
      84.79039923447887
     ],
     "I": [
      55.00644510399965,
      65.05987282834745
     ],
     "capability": [
      67.02511371513799,
      74.5041047187016
     ],
     "composite": {
      "A": [
       67.16941178445903,
       71.41294357282025
      ],
      "B": [
       64.31810039127373,
       69.97094141798148
      ],
      "C": [
       56.54608963400217,
       71.41294357282025
      ]
     },
     "seed": 106
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.93469131589123,
     "capability": 71.73894611323924,
     "intelligence": 60.54320091058725
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7626666666666667,
      "calibration": 85.2114699514098,
      "cc": 69.64845637202428,
      "cc_by_type": {
       "choice": 69.64845637202428
      },
      "cc_tiered": 70.19423744382789,
      "cc_tiered_open": 71.98506861847196,
      "cc_tiered_sealed": 68.40340626918382,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.64845637202428
     },
     "noul": {
      "accuracy": 0.8186666666666667,
      "calibration": 73.02895401264989,
      "cc": 63.733333333333334,
      "cc_by_type": {
       "noul": 63.733333333333334
      },
      "cc_tiered": 59.43497621117064,
      "cc_tiered_open": 56.49758454106281,
      "cc_tiered_sealed": 62.37236788127848,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 63.733333333333334
     },
     "score": {
      "accuracy": 0.6666666666666666,
      "calibration": 90.56364998361398,
      "cc": 52.778195902361034,
      "cc_by_type": {
       "score": 52.778195902361034
      },
      "cc_tiered": 52.000389076763234,
      "cc_tiered_open": 51.44344258419417,
      "cc_tiered_sealed": 52.557335569332295,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.778195902361034
     }
    },
    "raw_on_A": {
     "calibration": 79.9965213222889,
     "capability": 69.86117451490037,
     "intelligence": 59.725827707511854
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 66.35986297184556,
    "Emphasis on Accuracy 60:20:20": 63.904041615617075,
    "Emphasis on Cost 20:20:60": 61.96247920356028,
    "Emphasis on Speed 20:60:20": 74.51127093708331,
    "Intelligence only": 60.54320091058725
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 90.04368712409786,
    "cost": 57.71080894844077,
    "intelligence": 70.39456337588172,
    "speed": 87.62649746267333
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05156183229850985,
      "ece_score": 89.68763354029802,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 88.13304789226619,
      "score_before_time_limit": 89.68763354029802,
      "time_limit_failed_share": 0.017333333333333333
     },
     "noul": {
      "abstention_rate": 0.16800000000000004,
      "brier": 0.06086846046546967,
      "ece": 0.03262322669949693,
      "n": 375,
      "score": 92.47828421039287,
      "score_before_time_limit": 93.47535466010062,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.992,
      "interval90_width": 0.2688888888888889,
      "n": 375,
      "nrps": 0.04123358842551135,
      "score": 89.51972926963455,
      "score_before_time_limit": 90.97533462361235,
      "time_limit_failed_share": 0.016,
      "top_ece": 0.06962985955112075
     }
    },
    "score": 90.04368712409786
   },
   "capability": 80.21912524998979,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 738,068 input + 0 output tokens over 1,477 decisions = 499.707515234 tokens/decision. GPU-time cost USD 0.0256826422188204/1,000; old list-price cost USD 0.209877156398104/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0256826422188204/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.71455888
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.025682642218820436
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 23,
    "n_over_limit_answered_ok": 23,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "torchcast-decision-27b (Torchcast AI, Qwen3.8-27B fine-tune)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA H100 80GB, the author's own server in its own pinned environment, offline (Hub offline mode, all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "H100",
   "intelligence": {
    "I_open": 71.86444089391547,
    "I_sealed": 68.92468585784798,
    "base": 70.39456337588172,
    "excess": 0.36636739362925175,
    "gap": 2.9397550360674956,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 82.0423850709834,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 83.01352626612143,
       "judge": 84.17930125247197,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 62.8191856452726,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 73.91304347826086,
       "judge": 50,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 70.73175196549042,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 72.80894472531861,
       "hard": 75.13092960510187,
       "judge": 75.39366934082463,
       "standard": 53.90192424335207
      }
     },
     "sealed|choice": {
      "cc": 81.30477918983894,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 86.6972138053359,
       "hard": 76.07466063348416,
       "judge": 85.1024208566108,
       "standard": 83.37233649464221
      }
     },
     "sealed|noul": {
      "cc": 56.67251541954846,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 80,
       "hard": 39.80582524271845,
       "judge": 63.1578947368421,
       "standard": 69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 68.79676296415653,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 75.23124590065085,
       "hard": 60.91072527165571,
       "judge": 75.82659136309096,
       "standard": 70.80685428250935
      }
     }
    }
   },
   "jevbench_score": 74.00389040716802,
   "key": "torchcast-decision-27b",
   "last_measured_on": "2026-10-06",
   "licence": "cc-by-nc-4.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "torchcast-ai/torchcast-decision-27b@08645834899cb8e00d370b77a3f4e0baa3241dd1",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.970873786407767,
    "decisive_rate": 0.8328840970350404,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 91.379440941337,
     "cost": 30.341044859222123,
     "intelligence": 71.75187578095783,
     "speed": 87.72436167205926
    },
    "board_release": "v1.6.43",
    "capability": 81.56565836114741,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.87995157412942,
      21.459724866005526
     ],
     "B": [
      21.530802573949977,
      22.491279110132325
     ],
     "C": [
      20.87995157412942,
      21.459724866005526
     ]
    },
    "jevbench_score": 21.273291266043532,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 89,
    "rank": 123,
    "ranks": {
     "A": 123,
     "B": 115,
     "C": 104,
     "capability": 19
    },
    "scores": {
     "A": 21.273291266043532,
     "B": 22.135897876506068,
     "C": 21.273291266043532
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3445720289950259,
     "p50_s_raw": 0.09728601449751295,
     "p95_s_adjusted": 0.4900989957066485,
     "p95_s_raw": 0.17004949785332427
    },
    "views": {
     "Balanced 33:33:33": 18.95010611000118,
     "Emphasis on Accuracy 60:20:20": 21.366887812526656,
     "Emphasis on Cost 20:20:60": 14.822679909386718,
     "Emphasis on Speed 20:60:20": 22.70411049156899,
     "Intelligence only": 26.42130811221146
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a6 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 32,
   "ranked": true,
   "ranks": {
    "A": 32,
    "B": 27,
    "C": 27,
    "capability": 33
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-27b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.00389040716802,
    "B": 73.25271589899579,
    "C": 74.00389040716802
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.34793927800492386,
    "p50_s_raw": 0.09896963900246192,
    "p95_s_adjusted": 0.49641717158956455,
    "p95_s_raw": 0.17320858579478227
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.8-27B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.79008978256473,
      90.87788667710647
     ],
     "I": [
      65.2932742596741,
      74.33171049634291
     ],
     "capability": [
      76.43449921363415,
      81.80366414311386
     ],
     "composite": null,
     "seed": 120
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.04368712409786,
     "capability": 80.21912524998979,
     "intelligence": 70.39456337588172
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8586666666666667,
      "calibration": 88.13304789226619,
      "cc": 81.92548525525041,
      "cc_by_type": {
       "choice": 81.92548525525041
      },
      "cc_tiered": 81.67358213041118,
      "cc_tiered_open": 82.0423850709834,
      "cc_tiered_sealed": 81.30477918983894,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.92548525525041
     },
     "noul": {
      "accuracy": 0.8,
      "calibration": 92.47828421039287,
      "cc": 60.00000000000001,
      "cc_by_type": {
       "noul": 60.00000000000001
      },
      "cc_tiered": 59.74585053241053,
      "cc_tiered_open": 62.8191856452726,
      "cc_tiered_sealed": 56.67251541954846,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 60.00000000000001
     },
     "score": {
      "accuracy": 0.8133333333333334,
      "calibration": 89.51972926963455,
      "cc": 68.85226899393291,
      "cc_by_type": {
       "score": 68.85226899393291
      },
      "cc_tiered": 69.76425746482347,
      "cc_tiered_open": 70.73175196549042,
      "cc_tiered_sealed": 68.79676296415653,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 68.85226899393291
     }
    },
    "raw_on_A": {
     "calibration": 84.59019622902517,
     "capability": 79.48905505903151,
     "intelligence": 74.38791388903786
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.85599300657792,
    "Emphasis on Accuracy 60:20:20": 70.07042920804618,
    "Emphasis on Cost 20:20:60": 64.43212185684683,
    "Emphasis on Speed 20:60:20": 76.0229205329045,
    "Intelligence only": 70.39456337588172
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 95.28336264420759,
    "cost": 59.41752684953559,
    "intelligence": 88.35914237381022,
    "speed": 87.26135463003386
   },
   "base_model": "google/gemma-4-31B-it",
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04506978434694693,
      "ece_score": 90.98604313061061,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 354,
      "n_tvd": 0,
      "score": 90.98604313061061,
      "score_before_time_limit": 90.98604313061061,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.02126600295046343,
      "ece": 0.01164950382853796,
      "n": 375,
      "score": 97.67009923429241,
      "score_before_time_limit": 97.67009923429241,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9866666666666667,
      "interval90_width": 0.05422222222222222,
      "n": 375,
      "nrps": 0.014368384343134153,
      "score": 97.1939455677198,
      "score_before_time_limit": 97.1939455677198,
      "time_limit_failed_share": 0,
      "top_ece": 0.02087635215123492
     }
    },
    "score": 95.28336264420759
   },
   "capability": 91.8212525090089,
   "code_revision": "The author's own bundled HTTP server, unchanged: torchcast_decision.server (server.py sha256 81c10c49...) from the pinned HF repository @1e267fc9 (tag v1.0.0); hybrid attention (global layers non-causal) as shipped; independent source and kit reviews PASS (evaluator kit hash 15d49a98); no trust_remote_code, safetensors weights only. Run with the PyTorch SDPA attention kernel (TC_SLIDE=sdpa); the author's H100 FlashAttention-3 path was not used.",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10106.516129 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 560,794 input + 0 output tokens over 1,489 decisions = 376.624580255 tokens/decision. GPU-time cost USD 0.0225293729836294/1,000; old list-price cost USD 0.0338962122229684/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0225293729836294/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.03871607999999999
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02252937298362935
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "FIRST MEASUREMENT (paid evaluation, public). DRAW: measured on the 27 Oct 2026 regular draw (fresh sealed items; not yet equated to the reference scale, so its numbers are shown as published, offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; its use-case and language cells are measured on its own draw. FAMILY: a 31B full fine-tune of Gemma-4-31B-it; shares no weight shard with the earlier Torchcast rows (torchcast-decision-4b v1.1-v1.3 on Qwen3.5-4B, torchcast-decision-27b). CONTEXT: prompts longer than 16,384 tokens are cut by the model itself (its prompt.py keeps the head and the tail of the state around a \"...\" line) and answered (5 items on this draw; scored as answered). READOUT: one forward pass per question; softmax over the option-letter logits with stored temperatures (choice 0.8, yes/no 0.6, score 0.6); choice lists over 26 options are decided in groups (none on this draw). ATTENTION: the global attention layers attend to the whole prompt in both directions (author design, as in training). NOUL FLOOR: the author's server moves any yes/no probability with max(P, 1-P) < 0.801 to 0.801 / 0.199 on the same side (decision_config.json noul_floor 0.801, model.py lines 208-212); its code comment names JevBench's 0.2-0.8 abstention band. Kept unchanged as shipped and disclosed, as for earlier rows with the same floor; the argmax never changes, the Calibration axis includes its effect, and no variant without it was measured. HARDWARE: one Verda RTX PRO 6000 Blackwell Server Edition 96 GB (FIN-03, EU) with the PyTorch SDPA kernel (the author's H100 FlashAttention-3 path not used); latency reflects that card. ANSWERED: 1,500 of 1,500, HTTP 200 on every row, none over 10 s (maximum 6.235 s; measured p50 0.095 s, p95 0.202 s).",
   "display": "Torchcast Decision 31B v1.0",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-27",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.14",
    "results_sha256": "9e460d48571eeb00d94f554a985fdc2ebd52b63beb9aed7a4efe88762eb7b32e",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, 1x RTX PRO 6000 Blackwell Server Edition 96 GB (1RTXPRO6000.30V, FIN-03, Finland, EU, pay-as-you-go non-spot, USD 2.608/h), instance id c92731d5-54bc-4f2e-b781-5434544f75f0. Container with --network none (cut proven from inside before the sealed input), Hub offline, every credential unset, read-only weights, all 14 weight shards sha256-verified; hash-locked torch 2.13.0+cu130 / transformers 5.17.0 stack; driven one request at a time. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 27 Oct 2026 on the frozen v1.6-regular-20261027 self-hosted input (sha256 9b2f32ed...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 84.89591964246829,
    "I_sealed": 91.82236510515214,
    "base": 88.35914237381022,
    "excess": -9.499833105122102,
    "gap": -6.9264454626838585,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 86.95229422668578,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 83.01352626612143,
       "judge": 84.17930125247199,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 91.29951690821255,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 89.99999999999999,
       "standard": 88.88888888888889
      }
     },
     "open|score": {
      "cc": 76.43594779250654,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 58.47208828578886,
       "hard": 88.99270599413958,
       "judge": 86.03239291783133,
       "standard": 45.90969345461215
      }
     },
     "sealed|choice": {
      "cc": 90.82827710558826,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 98.03471994759252,
       "hard": 84.78766110289457,
       "judge": 94.82312338222607,
       "standard": 93.31401827501674
      }
     },
     "sealed|noul": {
      "cc": 93.32350160710908,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 100,
       "hard": 87.80487804878048,
       "judge": 95.55555555555554,
       "standard": 97.67441860465115
      }
     },
     "sealed|score": {
      "cc": 91.31531660275911,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 99.33913044897649,
       "hard": 84.86519551386306,
       "judge": 95.1494518555261,
       "standard": 94.45244897829201
      }
     }
    }
   },
   "jevbench_score": 79.83659491335965,
   "key": "torchcast-decision-31b-v1.0",
   "last_measured_on": "2026-10-27",
   "licence": "Apache-2.0 (weights and serving package; base Gemma-4-31B-it Apache-2.0), public, not gated",
   "listing": "unranked",
   "measured_in": "v1.7.14",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "torchcast-ai/torchcast-decision-31b@1e267fc9409654a3ac91c123265c21cddce73a3b",
   "new_in": "v1.7.14",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale: its first anchor runs used models weaker than this row, and stronger anchors are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.9733333333333334,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-31b",
   "scores": {
    "A": 79.83659491335965,
    "B": 81.40699258651951,
    "C": 79.83659491335965
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.33950119900000575,
    "p50_s_raw": 0.09475059950000286,
    "p95_s_adjusted": 0.5533797012999002,
    "p95_s_raw": 0.20168985064995013
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 100
      },
      "calibration_gen": {
       "accuracy": 0.8571428571428571,
       "cc": 83.98384623425146,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 51.95153870275438
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 83.98384623425146
      },
      "composite_gen": {
       "accuracy": 1,
       "cc": 96.6660597723805,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 89.99817931714156
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 96.6660597723805
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.83446214917221,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.50338644751659
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 99.83446214917221
      },
      "jaggedness": {
       "accuracy": 0.7272727272727273,
       "cc": 57.10254803634897,
       "cc_by_type": {
        "choice": 50.617283950617285,
        "noul": 33.33333333333333,
        "score": 87.35702682509631
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 57.10254803634897
      },
      "llm_router": {
       "accuracy": 0.9333333333333333,
       "cc": 87.36612311940509,
       "cc_by_type": {
        "choice": 97.3560517038778,
        "noul": 100,
        "score": 64.74231765433748
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 87.36612311940509
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": 51.61290322580645,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 51.61290322580645
      },
      "ml_usecase": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 100
      },
      "multilingual": {
       "accuracy": 0.9478827361563518,
       "cc": 92.78677223633593,
       "cc_by_type": {
        "choice": 90.29963354171157,
        "noul": 95.9016393442623,
        "score": 92.15904382303393
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 92.78677223633593
      },
      "negation_pair_gen": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 100
      },
      "option_order": {
       "accuracy": 0.9259259259259259,
       "cc": 90.47619047619048,
       "cc_by_type": {
        "choice": 90.47619047619048
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 90.47619047619048
      },
      "perturb_pair": {
       "accuracy": 0.9642857142857143,
       "cc": 93.16459811740611,
       "cc_by_type": {
        "choice": 100,
        "noul": 80,
        "score": 99.49379435221833
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 93.16459811740611
      },
      "usecase": {
       "accuracy": 0.9261538461538461,
       "cc": 90.55517987005096,
       "cc_by_type": {
        "choice": 87.60029175784098,
        "noul": 90.76923076923077,
        "score": 93.29601708308113
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 90.55517987005096
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.9523809523809523,
       "cc": 94.16620710078492,
       "cc_by_type": {
        "choice": 86.3013698630137,
        "noul": 100,
        "score": 96.19725143934107
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 94.16620710078492
      },
      "cs": {
       "accuracy": 0.918918918918919,
       "cc": 91.7032225886691,
       "cc_by_type": {
        "choice": 83.9572192513369,
        "noul": 100,
        "score": 91.15244851467044
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 91.7032225886691
      },
      "da": {
       "accuracy": 0.9444444444444444,
       "cc": 89.86278771725488,
       "cc_by_type": {
        "choice": 92.54658385093168,
        "noul": 100,
        "score": 77.04177930083294
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 89.86278771725488
      },
      "de": {
       "accuracy": 0.9791666666666666,
       "cc": 97.80350524596368,
       "cc_by_type": {
        "choice": 95.36917931566761,
        "noul": 100,
        "score": 98.04133642222345
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 97.80350524596368
      },
      "el": {
       "accuracy": 0.9285714285714286,
       "cc": 91.74198851309932,
       "cc_by_type": {
        "choice": 76.92307692307692,
        "noul": 100,
        "score": 98.30288861622103
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 91.74198851309932
      },
      "en": {
       "accuracy": 0.9261261261261261,
       "cc": 89.76476973985376,
       "cc_by_type": {
        "choice": 88.3976317799847,
        "noul": 91.80327868852459,
        "score": 89.09339875105198
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 89.76476973985376
      },
      "es": {
       "accuracy": 0.918918918918919,
       "cc": 85.59663012228636,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 77.77777777777777,
        "score": 79.01211258908131
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 85.59663012228636
      },
      "fi": {
       "accuracy": 0.9423076923076923,
       "cc": 93.44116062842919,
       "cc_by_type": {
        "choice": 93.14285714285714,
        "noul": 100,
        "score": 87.1806247424304
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 93.44116062842919
      },
      "fr": {
       "accuracy": 0.90625,
       "cc": 88.80492534444859,
       "cc_by_type": {
        "choice": 79.59183673469389,
        "noul": 100,
        "score": 86.82293929865187
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 88.80492534444859
      },
      "hi": {
       "accuracy": 0.918918918918919,
       "cc": 92.25293504142167,
       "cc_by_type": {
        "choice": 91.73553719008264,
        "noul": 100,
        "score": 85.02326793418236
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 92.25293504142167
      },
      "id": {
       "accuracy": 0.9512195121951219,
       "cc": 93.57686714904558,
       "cc_by_type": {
        "choice": 81.5668202764977,
        "noul": 100,
        "score": 99.16378117063903
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 93.57686714904558
      },
      "it": {
       "accuracy": 1,
       "cc": 98.81016475561341,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.43049426684027
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 98.81016475561341
      },
      "ja": {
       "accuracy": 0.9318181818181818,
       "cc": 91.32357047433516,
       "cc_by_type": {
        "choice": 76.68393782383419,
        "noul": 100,
        "score": 97.28677359917128
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 91.32357047433516
      },
      "ko": {
       "accuracy": 0.9534883720930233,
       "cc": 94.87590321106607,
       "cc_by_type": {
        "choice": 92.64705882352942,
        "noul": 100,
        "score": 91.98065080966879
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 94.87590321106607
      },
      "mixed": {
       "accuracy": 0.9375,
       "cc": 90.18005068021408,
       "cc_by_type": {
        "choice": 83.79837983798379,
        "noul": 89.99999999999999,
        "score": 96.74177220265847
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 90.18005068021408
      },
      "nl": {
       "accuracy": 0.9166666666666666,
       "cc": 77.10157564242293,
       "cc_by_type": {
        "choice": 92.92452830188678,
        "noul": 80,
        "score": 58.38019862538203
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 77.10157564242293
      },
      "no": {
       "accuracy": 1,
       "cc": 99.78012873958635,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 100,
        "score": 99.340386218759
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 99.78012873958635
      },
      "pl": {
       "accuracy": 0.9649122807017544,
       "cc": 92.30556287353848,
       "cc_by_type": {
        "choice": 95.21021820117083,
        "noul": 100,
        "score": 81.70647041944463
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 92.30556287353848
      },
      "pt": {
       "accuracy": 0.9148936170212766,
       "cc": 88.51809725913381,
       "cc_by_type": {
        "choice": 94.0099833610649,
        "noul": 83.33333333333333,
        "score": 88.21097508300323
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 88.51809725913381
      },
      "sv": {
       "accuracy": 0.9347826086956522,
       "cc": 89.82176808817444,
       "cc_by_type": {
        "choice": 86.59217877094974,
        "noul": 83.33333333333333,
        "score": 99.53979216024025
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 89.82176808817444
      },
      "tr": {
       "accuracy": 0.9782608695652174,
       "cc": 96.64142599399929,
       "cc_by_type": {
        "choice": 93.15849486887114,
        "noul": 100,
        "score": 96.76578311312673
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 96.64142599399929
      },
      "uk": {
       "accuracy": 0.9285714285714286,
       "cc": 86.52704302941265,
       "cc_by_type": {
        "choice": 89.04109589041097,
        "noul": 100,
        "score": 70.54003319782694
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 86.52704302941265
      },
      "zh": {
       "accuracy": 0.9512195121951219,
       "cc": 93.69137961538469,
       "cc_by_type": {
        "choice": 87.60330578512396,
        "noul": 100,
        "score": 93.47083306103013
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 93.69137961538469
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.924,
       "cc": 89.63332693415242,
       "cc_by_type": {
        "choice": 89.63332693415242
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 89.63332693415242
      },
      "noul": {
       "accuracy": 0.9733333333333334,
       "cc": 94.66666666666667,
       "cc_by_type": {
        "noul": 94.66666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 94.66666666666667
      },
      "score": {
       "accuracy": 0.9333333333333333,
       "cc": 90.307376376217,
       "cc_by_type": {
        "score": 90.307376376217
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 90.307376376217
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.875,
       "cc": 90.08835610693366,
       "cc_by_type": {
        "choice": 77.76130467012601,
        "noul": 100,
        "score": 92.50376365067495
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 90.08835610693366
      },
      "code_linting": {
       "accuracy": 0.9710144927536232,
       "cc": 95.82789508851037,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 100,
        "score": 87.4836852655311
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 95.82789508851037
      },
      "customer_support": {
       "accuracy": 0.9545454545454546,
       "cc": 92.94286247096998,
       "cc_by_type": {
        "choice": 92.619926199262,
        "noul": 86.66666666666667,
        "score": 99.54199454698124
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 92.94286247096998
      },
      "demand_forecasting": {
       "accuracy": 0.7272727272727273,
       "cc": 60.22099878636832,
       "cc_by_type": {
        "choice": 45.41319181197878,
        "noul": 69.23076923076923,
        "score": 66.01903531635698
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 60.22099878636832
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.9032258064516129,
       "cc": 86.95408530463777,
       "cc_by_type": {
        "choice": 87.33286418015481,
        "noul": 88.23529411764706,
        "score": 85.29409761611146
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 86.95408530463777
      },
      "feature_extraction": {
       "accuracy": 0.9166666666666666,
       "cc": 85.04834211690336,
       "cc_by_type": {
        "choice": 84.92462311557789,
        "noul": 71.42857142857142,
        "score": 98.79183180656077
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 85.04834211690336
      },
      "financial_crime": {
       "accuracy": 0.9682539682539683,
       "cc": 88.52265782904539,
       "cc_by_type": {
        "choice": 93.62041467304626,
        "noul": 100,
        "score": 71.9475588140899
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 88.52265782904539
      },
      "gaming": {
       "accuracy": 0.8793103448275862,
       "cc": 89.04226072473931,
       "cc_by_type": {
        "choice": 85.65737051792829,
        "noul": 100,
        "score": 81.46941165628961
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 89.04226072473931
      },
      "insurance_claims": {
       "accuracy": 0.9354838709677419,
       "cc": 89.78456541186615,
       "cc_by_type": {
        "choice": 86.6369710467706,
        "noul": 88.23529411764706,
        "score": 94.48143107118085
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 89.78456541186615
      },
      "knowledge_graphs": {
       "accuracy": 0.9552238805970149,
       "cc": 94.93110720846629,
       "cc_by_type": {
        "choice": 91.00449775112445,
        "noul": 100,
        "score": 93.78882387427439
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 94.93110720846629
      },
      "lead_generation": {
       "accuracy": 0.9866666666666667,
       "cc": 97.99072597827308,
       "cc_by_type": {
        "choice": 95.76868829337094,
        "noul": 100,
        "score": 98.2034896414483
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 97.99072597827308
      },
      "legal_compliance": {
       "accuracy": 0.9642857142857143,
       "cc": 95.78608156897958,
       "cc_by_type": {
        "choice": 87.96389167502507,
        "noul": 100,
        "score": 99.39435303191368
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 95.78608156897958
      },
      "llm_guardrails": {
       "accuracy": 0.9859154929577465,
       "cc": 98.65843517961063,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.97530553883189
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 98.65843517961063
      },
      "model_routing": {
       "accuracy": 0.9662162162162162,
       "cc": 92.01202566279487,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 76.0360769883846
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 92.01202566279487
      },
      "moderation_trust_safety": {
       "accuracy": 0.9830508474576272,
       "cc": 96.80238470593497,
       "cc_by_type": {
        "choice": 93.4640522875817,
        "noul": 100,
        "score": 96.94310183022324
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 96.80238470593497
      },
      "other": {
       "accuracy": 0.875,
       "cc": 85.07371570676834,
       "cc_by_type": {
        "choice": 80.02219755826859,
        "noul": 100,
        "score": 75.19894956203643
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 85.07371570676834
      },
      "recruiting": {
       "accuracy": 0.9692307692307692,
       "cc": 95.23217979048142,
       "cc_by_type": {
        "choice": 95.67411679884643,
        "noul": 92,
        "score": 98.02242257259786
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 95.23217979048142
      },
      "risk_assessment": {
       "accuracy": 0.9375,
       "cc": 92.82247096748746,
       "cc_by_type": {
        "choice": 87.44769874476985,
        "noul": 100,
        "score": 91.01971415769255
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 92.82247096748746
      },
      "scientific_discovery": {
       "accuracy": 1,
       "cc": 99.12946181285844,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.38838543857533
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 99.12946181285844
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.83243504813424,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.49730514440273
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 99.83243504813424
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      92.32739654839965,
      95.85357686379211
     ],
     "I": [
      84.30007735934731,
      91.07674939408005
     ],
     "capability": [
      88.88898295236517,
      92.99417064309486
     ],
     "composite": null,
     "seed": 20
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 95.28336264420759,
     "capability": 91.8212525090089,
     "intelligence": 88.35914237381022
    },
    "per_type": {
     "choice": {
      "accuracy": 0.924,
      "calibration": 90.98604313061061,
      "cc": 89.63332693415242,
      "cc_by_type": {
       "choice": 89.63332693415242
      },
      "cc_tiered": 88.89028566613702,
      "cc_tiered_open": 86.95229422668578,
      "cc_tiered_sealed": 90.82827710558826,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 89.63332693415242
     },
     "noul": {
      "accuracy": 0.9733333333333334,
      "calibration": 97.67009923429241,
      "cc": 94.66666666666667,
      "cc_by_type": {
       "noul": 94.66666666666667
      },
      "cc_tiered": 92.31150925766082,
      "cc_tiered_open": 91.29951690821255,
      "cc_tiered_sealed": 93.32350160710908,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 94.66666666666667
     },
     "score": {
      "accuracy": 0.9333333333333333,
      "calibration": 97.1939455677198,
      "cc": 90.307376376217,
      "cc_by_type": {
       "score": 90.307376376217
      },
      "cc_tiered": 83.87563219763283,
      "cc_tiered_open": 76.43594779250654,
      "cc_tiered_sealed": 91.31531660275911,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 90.307376376217
     }
    },
    "raw_on_A": {
     "calibration": 94.34413619198082,
     "capability": 90.24630088897172,
     "intelligence": 86.14846558596261
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 75.74356402084385,
    "Emphasis on Accuracy 60:20:20": 80.33132289180062,
    "Emphasis on Cost 20:20:60": 68.2431495455697,
    "Emphasis on Speed 20:60:20": 79.9654818676114,
    "Intelligence only": 88.35914237381022
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 83.11679203980711,
    "cost": 79.35029829231833,
    "intelligence": 54.20841827353793,
    "speed": 93.5282853940162
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07938734117954498,
      "ece_score": 84.12253176409101,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 84.12253176409101,
      "score_before_time_limit": 84.12253176409101,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.16388303423251918,
      "ece": 0.11132698482084283,
      "n": 375,
      "score": 77.73460303583144,
      "score_before_time_limit": 77.73460303583144,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.896,
      "interval90_width": 0.3931111111111111,
      "n": 375,
      "nrps": 0.10433692214009276,
      "score": 87.49324131949886,
      "score_before_time_limit": 87.49324131949886,
      "time_limit_failed_share": 0,
      "top_ece": 0.07289912573496503
     }
    },
    "score": 83.11679203980711
   },
   "capability": 68.66260515667253,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 737,326 input + 0 output tokens over 1,485 decisions = 496.515824916 tokens/decision. GPU-time cost USD 0.00487891655385447/1,000; old list-price cost USD 0.0148954747474747/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00487891655385447/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.01965778
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004878916553854475
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Torchcast Decision 4B (v1.1)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container (--network none); author's torchcast_decision server unchanged at tag v1.1.0 with the bundled decision_config.json (bf16, one forward pass reading option letters, calibration temperature 0.9, yes/no commit floor 0.801) on the documented stack torch 2.9.1+cu128 / transformers 5.8.1 / flash-linear-attention 0.5.2 / causal-conv1d 1.7.0 (fast kernels active, CUDA graphs on) paid evaluation (public). Qwen3.5-4B fine-tune served by the author's own torchcast_decision package. Runtime: official pytorch/pytorch:2.9.1-cuda12.8-cudnn9-devel image (digest-pinned) plus an offline hash-locked overlay of the documented packages (Python 3.11 instead of the documented 3.12). decision_config.json sets a disclosed yes/no commit floor of 0.801: a yes/no probability with max(P, 1-P) below 0.801 is moved to 0.801 on its own side (answer unchanged; same mechanism as h2o-lightning-4b). Prompts over 16,384 tokens keep the head and the tail of the state (documented). Output tokens are 0 (no generation).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 59.789857618232716,
    "I_sealed": 48.626978928843144,
    "base": 54.20841827353793,
    "excess": 1.3233519190338612,
    "gap": 11.162878689389572,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 75.49631276628199,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 62.25228059138094,
       "judge": 76.26895187870798,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 61.89855072463768,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 30.000000000000004,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 41.974709363778516,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.04587734174547,
       "hard": 42.40482365626291,
       "judge": 40.031546412698795,
       "standard": 34.993641216445845
      }
     },
     "sealed|choice": {
      "cc": 62.83817359092292,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 79.03430749682339,
       "hard": 55.76500520411702,
       "judge": 62.97747306562194,
       "standard": 68.67749419953597
      }
     },
     "sealed|noul": {
      "cc": 50.59073813474225,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 68.96551724137932,
       "hard": 54.54545454545454,
       "judge": 26.15384615384615,
       "standard": 70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 32.45202506086426,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 52.45476052442795,
       "hard": 33.44972503305952,
       "judge": 17.565215027813473,
       "standard": 42.78547243426806
      }
     }
    }
   },
   "jevbench_score": 74.38693697091249,
   "key": "torchcast-decision-4b-v1.1",
   "last_measured_on": "2026-10-12",
   "licence": "CC BY-NC 4.0 (weights); base Qwen/Qwen3.5-4B Apache-2.0; serving package Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.12",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "torchcast-ai/torchcast-decision-4b@f4a34e7aedbf22917f6c85b8c4e03a3da49c1baf",
   "noul_decisive": {
    "acc_among_decisive": 0.784,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.11679203980711,
     "cost": 64.8083695127977,
     "intelligence": 54.20841827353793,
     "speed": 93.62765623388151
    },
    "board_release": "v1.6.43",
    "capability": 68.66260515667253,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 30,
    "capability_rank": 51,
    "composite_ci95": {
     "A": [
      64.29630519793719,
      72.43041609849763
     ],
     "B": [
      59.80423893427847,
      69.05468745077442
     ],
     "C": [
      44.65021194301194,
      68.26018700789584
     ]
    },
    "jevbench_score": 70.68474930403303,
    "measured_in": "v1.6.12",
    "method": "v1.6",
    "open_board_rank": 9,
    "rank": 23,
    "ranks": {
     "A": 23,
     "B": 37,
     "C": 43,
     "capability": 86
    },
    "scores": {
     "A": 70.68474930403303,
     "B": 66.6341381514443,
     "C": 57.69745963223827
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.1927332690451294,
     "p50_s_raw": 0.02136663452256471,
     "p95_s_adjusted": 0.2250493116211146,
     "p95_s_raw": 0.037524655810557306
    },
    "views": {
     "Balanced 33:33:33": 67.3279317302704,
     "Emphasis on Accuracy 60:20:20": 61.38534303539167,
     "Emphasis on Cost 20:20:60": 66.2969580344183,
     "Emphasis on Speed 20:60:20": 75.8503894110146,
     "Intelligence only": 54.20841827353793
    }
   },
   "rank": 27,
   "ranked": true,
   "ranks": {
    "A": 27,
    "B": 61,
    "C": 84,
    "capability": 110
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.38693697091249,
    "B": 69.23270630649412,
    "C": 60.71942442045547
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.19473725310526788,
    "p50_s_raw": 0.02236862655263394,
    "p95_s_adjusted": 0.22788850581273434,
    "p95_s_raw": 0.03894425290636718
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 60.2110405969951,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": 13.9664551243186
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 60.2110405969951
      },
      "calibration_gen": {
       "accuracy": 0.6237623762376238,
       "cc": 49.1657668398499,
       "cc_by_type": {
        "choice": 52.27995758218452,
        "noul": 68,
        "score": 27.217342937365164
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 49.1657668398499
      },
      "composite_gen": {
       "accuracy": 0.801980198019802,
       "cc": 59.517608397224535,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 25.92592592592593,
        "score": 59.130964306398084
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 59.517608397224535
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 87.56706701867559,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 62.701201056026754
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 87.56706701867559
      },
      "jaggedness": {
       "accuracy": 0.5388888888888889,
       "cc": 35.249975319792384,
       "cc_by_type": {
        "choice": 45.13955726660251,
        "noul": 51.35135135135136,
        "score": 9.25901734142327
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 35.249975319792384
      },
      "llm_router": {
       "accuracy": 0.8083067092651757,
       "cc": 69.03721475082187,
       "cc_by_type": {
        "choice": 70.67887646385489,
        "noul": 85.56701030927836,
        "score": 50.86575747933239
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 69.03721475082187
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": 27.405528195442695,
       "cc_by_type": {
        "choice": -3.2258064516129026,
        "noul": 100,
        "score": -14.557608962059021
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 27.405528195442695
      },
      "ml_usecase": {
       "accuracy": 0.5555555555555556,
       "cc": 16.794675809772134,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": 25,
        "score": -0.3891684469722545
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 16.794675809772134
      },
      "multilingual": {
       "accuracy": 0.6727272727272727,
       "cc": 48.38591044723288,
       "cc_by_type": {
        "choice": 61.85197261167266,
        "noul": 44.44444444444444,
        "score": 38.86131428558153
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 48.38591044723288
      },
      "negation_pair_gen": {
       "accuracy": 0.7674418604651163,
       "cc": 50.41309466275076,
       "cc_by_type": {
        "choice": 90,
        "noul": 19.999999999999996,
        "score": 41.23928398825228
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 50.41309466275076
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 80.37317882017604,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100,
        "score": 62.140557481549116
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.37317882017604
      },
      "usecase": {
       "accuracy": 0.6,
       "cc": 40.995529717776705,
       "cc_by_type": {
        "choice": 57.10784313725491,
        "noul": 33.33333333333333,
        "score": 32.54541268274188
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 40.995529717776705
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.375,
       "cc": -0.033285890015636234,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": 50,
        "score": -54.86176243195167
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 56.06521080351538,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 34.862299077212796
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 56.06521080351538
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 61.46528196168856,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 75,
        "score": 41.25425296471171
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 61.46528196168856
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 59.346774360020476,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 33.33333333333333,
        "score": 94.98631935566664
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 59.346774360020476
      },
      "en": {
       "accuracy": 0.7072111207645526,
       "cc": 53.8082139855539,
       "cc_by_type": {
        "choice": 66.1707814158968,
        "noul": 58.273381294964025,
        "score": 36.98047924580088
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 53.8082139855539
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 61.54934905483835,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 4.2239036082344805
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 61.54934905483835
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 37.48599809267048,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 11.911546190579736
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 37.48599809267048
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 40.01622520990754,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 0,
        "score": 41.22514621795791
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 40.01622520990754
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 33.23647842158908,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 25,
        "score": 24.988764873705794
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 33.23647842158908
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 80.34560570535994,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 55.06643519869871
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 80.34560570535994
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 45.84126362278722,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 39.99999999999999,
        "score": 54.71203550696373
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 45.84126362278722
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 5.848578443484641,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 17.545735330453926
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 5.848578443484641
      },
      "mixed": {
       "accuracy": 1,
       "cc": 97.57222944560151,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 92.71668833680451
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 97.57222944560151
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 53.346460809740314,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 78.65410104393959
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 53.346460809740314
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 71.90902061780149,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80,
        "score": 43.34449731679848
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 71.90902061780149
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 65.05242711174127,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": 48.19204857152315
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 65.05242711174127
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 40.49448282015345,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": 60.00000000000001,
        "score": 38.07919314131141
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 40.49448282015345
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": 29.93255038880676,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 100,
        "score": -44.984957529231906
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 29.93255038880676
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 76.16771943867207,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 28.503158316016197
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 76.16771943867207
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 22.18571449165178,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": -15.928732231259346
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 22.18571449165178
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7333333333333333,
       "cc": 65.55617214259745,
       "cc_by_type": {
        "choice": 65.55617214259745
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 65.55617214259745
      },
      "noul": {
       "accuracy": 0.784,
       "cc": 56.800000000000004,
       "cc_by_type": {
        "noul": 56.800000000000004
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 56.800000000000004
      },
      "score": {
       "accuracy": 0.5653333333333334,
       "cc": 36.55933643117994,
       "cc_by_type": {
        "score": 36.55933643117994
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 36.55933643117994
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 19.320960055963663,
       "cc_by_type": {
        "choice": 36,
        "noul": 25,
        "score": -3.0371198321090143
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 19.320960055963663
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 34.22871764612634,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 33.33333333333333,
        "score": 6.619900350387297
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 34.22871764612634
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 11.281778554363948,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": -67.3741765320301
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 11.281778554363948
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 29.702151966686078,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": 33.33333333333333,
        "score": 37.64446759596467
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 29.702151966686078
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 36.94365711968195,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 35.46865251846613
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 36.94365711968195
      },
      "feature_extraction": {
       "accuracy": 0.47368421052631576,
       "cc": -34.15263657120354,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": 33.33333333333333,
        "score": -160.31954493373638
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 56.27374050728449,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 71.42857142857142,
        "score": 17.00049323053695
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 56.27374050728449
      },
      "gaming": {
       "accuracy": 0.5,
       "cc": 22.222092708926684,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": 0,
        "score": 29.384048161623255
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 22.222092708926684
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 25.03420356730068,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 19.999999999999996,
        "score": 38.43594403523537
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 25.03420356730068
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 25.182427133062486,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 33.33333333333333,
        "score": 32.14970823716035
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 25.182427133062486
      },
      "lead_generation": {
       "accuracy": 0.68,
       "cc": 53.80625460601258,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 55.55555555555556,
        "score": 29.119022215970546
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 53.80625460601258
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 17.361266487762062,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": 15.524659678339948
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 17.361266487762062
      },
      "llm_guardrails": {
       "accuracy": 0.7037037037037037,
       "cc": 48.34107386804876,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 36.5395363834169
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 48.34107386804876
      },
      "model_routing": {
       "accuracy": 0.8387096774193549,
       "cc": 69.88393499048401,
       "cc_by_type": {
        "choice": 82.46753246753246,
        "noul": 90.90909090909092,
        "score": 36.27518159482861
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 69.88393499048401
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 9.85366085849237,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 52.593612134018365
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 9.85366085849237
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 50.4395712119974,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 51.41156414667002
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 50.4395712119974
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 23.726968533929526,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": -14.040769275058707
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 23.726968533929526
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 19.828044012512837,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": -33.333333333333336,
        "score": 21.802972617248663
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 19.828044012512837
      },
      "search_retrieval": {
       "accuracy": 0.9166666666666666,
       "cc": 90.9303532843901,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 83.4672164368002
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 90.9303532843901
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.22893804404913,
      84.7914607783849
     ],
     "I": [
      48.8894404140232,
      58.33903054464181
     ],
     "capability": [
      63.688569202590465,
      70.97559770691103
     ],
     "composite": null,
     "seed": 76
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.11679203980711,
     "capability": 68.66260515667253,
     "intelligence": 54.20841827353793
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7333333333333333,
      "calibration": 84.12253176409101,
      "cc": 65.55617214259745,
      "cc_by_type": {
       "choice": 65.55617214259745
      },
      "cc_tiered": 69.16724317860246,
      "cc_tiered_open": 75.49631276628199,
      "cc_tiered_sealed": 62.83817359092292,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.55617214259745
     },
     "noul": {
      "accuracy": 0.784,
      "calibration": 77.73460303583144,
      "cc": 56.800000000000004,
      "cc_by_type": {
       "noul": 56.800000000000004
      },
      "cc_tiered": 56.24464442968997,
      "cc_tiered_open": 61.89855072463768,
      "cc_tiered_sealed": 50.59073813474225,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 56.800000000000004
     },
     "score": {
      "accuracy": 0.5653333333333334,
      "calibration": 87.49324131949886,
      "cc": 36.55933643117994,
      "cc_by_type": {
       "score": 36.55933643117994
      },
      "cc_tiered": 37.21336721232139,
      "cc_tiered_open": 41.974709363778516,
      "cc_tiered_sealed": 32.45202506086426,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 36.55933643117994
     }
    },
    "raw_on_A": {
     "calibration": 82.03015024423762,
     "capability": 68.44650938996614,
     "intelligence": 54.862868535694645
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 71.87071488065399,
    "Emphasis on Accuracy 60:20:20": 63.5839007623379,
    "Emphasis on Cost 20:20:60": 74.68671180942998,
    "Emphasis on Speed 20:60:20": 79.20726284827764,
    "Intelligence only": 54.20841827353793
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 84.9508191193537,
    "cost": 79.35029829231833,
    "intelligence": 60.262524104890716,
    "speed": 90.89431110061412
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.08466140495406262,
      "ece_score": 83.06771900918748,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 83.06771900918748,
      "score_before_time_limit": 83.06771900918748,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.12193405070480291,
      "ece": 0.10004788600754745,
      "n": 375,
      "score": 79.99042279849051,
      "score_before_time_limit": 79.99042279849051,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9466666666666667,
      "interval90_width": 0.4382222222222224,
      "n": 375,
      "nrps": 0.08850262773720094,
      "score": 91.7943155503831,
      "score_before_time_limit": 91.7943155503831,
      "time_limit_failed_share": 0,
      "top_ece": 0.03780553062756855
     }
    },
    "score": 84.9508191193537
   },
   "capability": 72.6066716121222,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 737,326 input + 0 output tokens over 1,485 decisions = 496.515824916 tokens/decision. GPU-time cost USD 0.00487891655385447/1,000; old list-price cost USD 0.0148954747474747/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00487891655385447/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.019657780000000003
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004878916553854475
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Torchcast Decision 4B (v1.2)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-18",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Verda FIN-01 A100 SXM4 80GB GPU VM, offline read-only container (--network none); author's torchcast_decision server unchanged at tag v1.2.0 with the bundled decision_config.json (bf16, one forward pass reading option letters, non-causal full attention as trained, calibration temperature 1.0, yes/no commit floor 0.801) on the documented stack torch 2.9.1+cu128 / transformers 5.8.1 / flash-linear-attention 0.5.2 / causal-conv1d 1.7.0 (fast kernels active; no CUDA graphs in non-causal mode)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 62.74872292249266,
    "I_sealed": 57.77632528728878,
    "base": 60.262524104890716,
    "excess": -4.867129135151828,
    "gap": 4.972397635203883,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 72.16814944751415,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 92.21621621621622,
       "hard": 69.80182447310474,
       "judge": 80.22412656558998,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 73.59903381642512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 42.47898550353872,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.38410762065289,
       "hard": 52.07017376518772,
       "judge": 43.985586059358674,
       "standard": 19.084147087953696
      }
     },
     "sealed|choice": {
      "cc": 70.81428900718828,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 86.6581956797967,
       "hard": 69.81612119810339,
       "judge": 63.85896180215475,
       "standard": 75.32166209660409
      }
     },
     "sealed|noul": {
      "cc": 63.36387030365415,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 60.00000000000001,
       "judge": 60.00000000000001,
       "standard": 67.16417910447761
      }
     },
     "sealed|score": {
      "cc": 39.15081655102391,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 68.46129282933187,
       "hard": 34.48432889360359,
       "judge": 35.38697723076133,
       "standard": 39.474312707104446
      }
     }
    }
   },
   "jevbench_score": 76.9679054341264,
   "key": "torchcast-decision-4b-v1.2",
   "last_measured_on": "2026-10-18",
   "licence": "Apache-2.0 (weights, from v1.2); base Qwen/Qwen3.5-4B Apache-2.0; serving package Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.29",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "torchcast-ai/torchcast-decision-4b@d54b248f2643df847979ca15638ee502ec79b788",
   "noul_decisive": {
    "acc_among_decisive": 0.8373333333333334,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 84.9508191193537,
     "cost": 64.8083695127977,
     "intelligence": 60.262524104890716,
     "speed": 91.12622136196717
    },
    "board_release": "v1.6.43",
    "capability": 72.6066716121222,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 16,
    "capability_rank": 33,
    "composite_ci95": {
     "A": [
      70.29693613333056,
      74.3666613117086
     ],
     "B": [
      66.53046829296893,
      72.03744728597768
     ],
     "C": [
      58.79194113894132,
      74.3666613117086
     ]
    },
    "jevbench_score": 73.03338420137015,
    "measured_in": "v1.6.29",
    "method": "v1.6",
    "open_board_rank": 4,
    "rank": 14,
    "ranks": {
     "A": 14,
     "B": 20,
     "C": 14,
     "capability": 59
    },
    "scores": {
     "A": 73.03338420137015,
     "B": 70.06379430073542,
     "C": 73.03338420137015
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.2749862310000367,
     "p50_s_raw": 0.06249311550001835,
     "p95_s_adjusted": 0.28058659270005815,
     "p95_s_raw": 0.06529329635002909
    },
    "views": {
     "Balanced 33:33:33": 69.77075738979846,
     "Emphasis on Accuracy 60:20:20": 65.62878799360888,
     "Emphasis on Cost 20:20:60": 67.69731928870637,
     "Emphasis on Speed 20:60:20": 76.98758510619885,
     "Intelligence only": 60.262524104890716
    }
   },
   "rank": 7,
   "ranked": true,
   "ranks": {
    "A": 7,
    "B": 30,
    "C": 6,
    "capability": 78
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 76.9679054341264,
    "B": 72.9248066067173,
    "C": 76.9679054341264
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2755932589999134,
    "p50_s_raw": 0.06279662949995668,
    "p95_s_adjusted": 0.2953250589999356,
    "p95_s_raw": 0.0726625294999678
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 78.58208761447007,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 44.07959617674351
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 78.58208761447007
      },
      "calibration_gen": {
       "accuracy": 0.504950495049505,
       "cc": 29.652491168192714,
       "cc_by_type": {
        "choice": 33.19194061505833,
        "noul": 52,
        "score": 3.765532889519818
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 29.652491168192714
      },
      "composite_gen": {
       "accuracy": 0.8118811881188119,
       "cc": 69.68323903439195,
       "cc_by_type": {
        "choice": 86.9918699186992,
        "noul": 62.962962962962955,
        "score": 59.09488422151368
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 69.68323903439195
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 92.44612065592298,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 77.33836196776895
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 92.44612065592298
      },
      "jaggedness": {
       "accuracy": 0.6055555555555555,
       "cc": 40.56092887250238,
       "cc_by_type": {
        "choice": 55.24542829643889,
        "noul": 45.945945945945944,
        "score": 20.49141237512231
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 40.56092887250238
      },
      "llm_router": {
       "accuracy": 0.8626198083067093,
       "cc": 75.58422268224218,
       "cc_by_type": {
        "choice": 83.35828123624195,
        "noul": 93.81443298969072,
        "score": 49.5799538207939
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 75.58422268224218
      },
      "long_state": {
       "accuracy": 0.3333333333333333,
       "cc": -5.097572874998855,
       "cc_by_type": {
        "choice": 9.677419354838706,
        "noul": 0,
        "score": -24.970137979835272
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4444444444444444,
       "cc": 4.073147067528581,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": 0,
        "score": -1.1826206530843653
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 4.073147067528581
      },
      "multilingual": {
       "accuracy": 0.7672727272727272,
       "cc": 64.00830885734634,
       "cc_by_type": {
        "choice": 71.63351809585916,
        "noul": 72.22222222222223,
        "score": 48.169186253957605
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 64.00830885734634
      },
      "negation_pair_gen": {
       "accuracy": 0.7790697674418605,
       "cc": 47.57400302837292,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 22.722009085118767
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 47.57400302837292
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 80.37903990692341,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100,
        "score": 65.16114374479423
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.37903990692341
      },
      "usecase": {
       "accuracy": 0.7,
       "cc": 57.6090573165931,
       "cc_by_type": {
        "choice": 68.1372549019608,
        "noul": 58.33333333333333,
        "score": 46.35658371448517
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 57.6090573165931
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 9.22238439439781,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50,
        "score": -46.142370626330376
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 9.22238439439781
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 45.79680121584361,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": 24.7943975031376
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 45.79680121584361
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 77.70007818419383,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100,
        "score": 72.92324340213901
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 77.70007818419383
      },
      "el": {
       "accuracy": 0.8,
       "cc": 74.26271122284257,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 89.64102007448673
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 74.26271122284257
      },
      "en": {
       "accuracy": 0.7471763683753258,
       "cc": 60.10371398764253,
       "cc_by_type": {
        "choice": 72.98055918282665,
        "noul": 66.18705035971225,
        "score": 41.143532420388674
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 60.10371398764253
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 64.64149013424526,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 3.712398624595481
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 64.64149013424526
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 58.4795453997909,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 74.89218811194102
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 58.4795453997909
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 77.57866686911653,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 53.91247119558486
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 77.57866686911653
      },
      "hi": {
       "accuracy": 0.6666666666666666,
       "cc": 38.57772998083417,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 25,
        "score": 41.01251955144107
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 38.57772998083417
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 71.19687350509501,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 27.620238597903914
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 71.19687350509501
      },
      "ja": {
       "accuracy": 0.8,
       "cc": 62.53605931537678,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 100,
        "score": 16.20230026543141
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 62.53605931537678
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 61.26924456890534,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100,
        "score": 28.252178151160468
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 61.26924456890534
      },
      "mixed": {
       "accuracy": 1,
       "cc": 96.12707554673155,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 88.38122664019465
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 96.12707554673155
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 46.541321476730594,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 33.33333333333333,
        "score": 58.238683044910424
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 46.541321476730594
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.7948717948717948,
       "cc": 76.23616141039446,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 80,
        "score": 63.94335515797137
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 76.23616141039446
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 68.46010565834278,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 37.20117731680793
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 68.46010565834278
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 65.46355665099959,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 61.92258484661579
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 65.46355665099959
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 37.91534536977069,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 50,
        "score": -14.514833455905318
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 37.91534536977069
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 67.99341252934869,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 36.23830210417509
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 67.99341252934869
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 31.75490779026978,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": 12.77884766459465
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 31.75490779026978
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7853333333333333,
       "cc": 72.27271857479094,
       "cc_by_type": {
        "choice": 72.27271857479094
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 72.27271857479094
      },
      "noul": {
       "accuracy": 0.8373333333333334,
       "cc": 67.46666666666667,
       "cc_by_type": {
        "noul": 67.46666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 67.46666666666667
      },
      "score": {
       "accuracy": 0.584,
       "cc": 41.326578005993696,
       "cc_by_type": {
        "score": 41.326578005993696
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 41.326578005993696
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 54.51220458804392,
       "cc_by_type": {
        "choice": 52,
        "noul": 75,
        "score": 36.536613764131765
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 54.51220458804392
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 19.414032579676796,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": 11.111111111111116,
        "score": 3.031607745931697
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 19.414032579676796
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 47.98302937571595,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 42.72957593202591
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 47.98302937571595
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 50.4289331415679,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 100,
        "score": 21.462238021194935
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 50.4289331415679
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 59.47084257633375,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 59.57194801885635
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 59.47084257633375
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -56.08570821762897,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": 66.66666666666667,
        "score": -259.45209320634604
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 50.752697382203884,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 29.008792426723684
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 50.752697382203884
      },
      "gaming": {
       "accuracy": 0.5909090909090909,
       "cc": 42.737556232348936,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": 50,
        "score": 30.477477059416145
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 42.737556232348936
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 34.180989510345235,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 45.04296853103571
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 34.180989510345235
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 53.74050987098581,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 66.66666666666667,
        "score": 33.09876016256486
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 53.74050987098581
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 61.064515449092106,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 39.26589776846497
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 61.064515449092106
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 11.797581376060576,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -1.1663956567645073
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 11.797581376060576
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 50.15312163699627,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 19.999999999999996,
        "score": 53.49199446953005
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 50.15312163699627
      },
      "model_routing": {
       "accuracy": 0.8663594470046083,
       "cc": 74.8931310594956,
       "cc_by_type": {
        "choice": 90.64935064935067,
        "noul": 96.96969696969697,
        "score": 37.06034555943917
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.8931310594956
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 49.724431860882,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 72.20592514118727
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 49.724431860882
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 82.1798596706628,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100,
        "score": 63.25267093399397
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 82.1798596706628
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 39.6143521594331,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 50,
        "score": 13.178031847757444
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 39.6143521594331
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 47.91103234199653,
       "cc_by_type": {
        "choice": 85.5072463768116,
        "noul": 33.33333333333333,
        "score": 24.892517315844675
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 47.91103234199653
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 92.8233635346338,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 89.1462471875313
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 92.8233635346338
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.96169960479254,
      86.52145783591912
     ],
     "I": [
      54.797400191098326,
      64.34833891333052
     ],
     "capability": [
      68.1468487749732,
      74.61557349985053
     ],
     "composite": null,
     "seed": 77
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.9508191193537,
     "capability": 72.6066716121222,
     "intelligence": 60.262524104890716
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7853333333333333,
      "calibration": 83.06771900918748,
      "cc": 72.27271857479094,
      "cc_by_type": {
       "choice": 72.27271857479094
      },
      "cc_tiered": 71.49121922735122,
      "cc_tiered_open": 72.16814944751415,
      "cc_tiered_sealed": 70.81428900718828,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 72.27271857479094
     },
     "noul": {
      "accuracy": 0.8373333333333334,
      "calibration": 79.99042279849051,
      "cc": 67.46666666666667,
      "cc_by_type": {
       "noul": 67.46666666666667
      },
      "cc_tiered": 68.48145206003963,
      "cc_tiered_open": 73.59903381642512,
      "cc_tiered_sealed": 63.36387030365415,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 67.46666666666667
     },
     "score": {
      "accuracy": 0.584,
      "calibration": 91.7943155503831,
      "cc": 41.326578005993696,
      "cc_by_type": {
       "score": 41.326578005993696
      },
      "cc_tiered": 40.814901027281316,
      "cc_tiered_open": 42.47898550353872,
      "cc_tiered_sealed": 39.15081655102391,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 41.326578005993696
     }
    },
    "raw_on_A": {
     "calibration": 83.43579348904761,
     "capability": 72.66452279180986,
     "intelligence": 61.893252094572105
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 74.63021342876776,
    "Emphasis on Accuracy 60:20:20": 68.13259051689728,
    "Emphasis on Cost 20:20:60": 76.4492201089296,
    "Emphasis on Speed 20:60:20": 80.38355947036793,
    "Intelligence only": 60.262524104890716
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Torchcast AI",
   "axes": {
    "calibration": 87.84716845590987,
    "cost": 79.35029829231833,
    "intelligence": 67.14748494108278,
    "speed": 92.44891117449383
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06325270940387061,
      "ece_score": 87.34945811922587,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.34945811922587,
      "score_before_time_limit": 87.34945811922587,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.1069128488645628,
      "ece": 0.06240741177362208,
      "n": 375,
      "score": 87.51851764527558,
      "score_before_time_limit": 87.51851764527558,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.928,
      "interval90_width": 0.3104444444444443,
      "n": 375,
      "nrps": 0.0771279973557552,
      "score": 88.67352960322816,
      "score_before_time_limit": 88.67352960322816,
      "time_limit_failed_share": 0,
      "top_ece": 0.07470070528984067
     }
    },
    "score": 87.84716845590987
   },
   "capability": 77.49732669849632,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 737,326 input + 0 output tokens over 1,485 decisions = 496.515824916 tokens/decision. GPU-time cost USD 0.00487891655385447/1,000; old list-price cost USD 0.0148954747474747/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00487891655385447/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.019657780000000003
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004878916553854475
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Torchcast Decision 4B (v1.3)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM (1A100.22V, FIN-02, Finland, EU, pay-as-you-go), 1x NVIDIA A100 SXM4 80GB; offline read-only container (network none); author's torchcast_decision server unchanged at tag v1.3.0 with the bundled decision_config.json (bf16, one forward pass reading option letters, non-causal full attention as trained, calibration temperature 0.9, yes/no commit floor 0.801) on the documented stack torch 2.9.1+cu128 / transformers 5.8.1 / flash-linear-attention 0.5.2 / causal-conv1d 1.7.0 (fast kernels active; CUDA graphs with a key-padding mask in non-causal mode). Driven by the unchanged self-hosted runner on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...), measured 23 Oct 2026; raw latency p50 0.034 s, p95 0.056 s, maximum 7.73 s. The standard self-hosted x2 + 0.15 s adjustment applies (assumption, not measured).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 69.73993545583312,
    "I_sealed": 64.55503442633244,
    "base": 67.14748494108278,
    "excess": -4.654625740855032,
    "gap": 5.184901029500679,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 81.67361775115984,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 77.35136835482857,
       "judge": 84.17930125247199,
       "standard": 77.39640016743407
      }
     },
     "open|noul": {
      "cc": 80.09316770186336,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 91.30434782608697,
       "judge": 50,
       "standard": 100
      }
     },
     "open|score": {
      "cc": 47.45302091447617,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.35515078027366,
       "hard": 59.61562740569072,
       "judge": 50.73096446053644,
       "standard": 23.75982768005792
      }
     },
     "sealed|choice": {
      "cc": 72.9759407483175,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 90.47013977128337,
       "hard": 68.77529779114143,
       "judge": 67.384916748286,
       "standard": 81.01666315123391
      }
     },
     "sealed|noul": {
      "cc": 70.89886952985255,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 65.45454545454545,
       "judge": 78.46153846153847,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 49.79029300082726,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 70.76877798198868,
       "hard": 45.662863577821,
       "judge": 48.12725276024139,
       "standard": 50.05046971713786
      }
     }
    }
   },
   "jevbench_score": 80.49078635844445,
   "key": "torchcast-decision-4b-v1.3",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 (weights, from v1.2); base Qwen/Qwen3.5-4B Apache-2.0; serving package Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.42",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "torchcast-ai/torchcast-decision-4b@004acea2e23056bc79415cfa69b9f3ea4254e6d0",
   "noul_decisive": {
    "acc_among_decisive": 0.8613333333333333,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": true,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 87.84716845590987,
     "cost": 64.8083695127977,
     "intelligence": 67.14748494108278,
     "speed": 92.63835078333227
    },
    "board_release": "v1.6.43",
    "capability": 77.49732669849632,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 11,
    "capability_rank": 20,
    "composite_ci95": {
     "A": [
      73.60377007858608,
      77.28836347482343
     ],
     "B": [
      70.78560945953136,
      75.84827959180653
     ],
     "C": [
      73.59888426077293,
      77.28836347482343
     ]
    },
    "jevbench_score": 76.18936170949911,
    "measured_in": "v1.6.42",
    "method": "v1.6",
    "open_board_rank": 1,
    "rank": 3,
    "ranks": {
     "A": 3,
     "B": 7,
     "C": 3,
     "capability": 36
    },
    "scores": {
     "A": 76.18936170949911,
     "B": 74.19128695067,
     "C": 76.18936170949911
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21561166599999523,
     "p50_s_raw": 0.03280583299999762,
     "p95_s_adjusted": 0.25263450489999856,
     "p95_s_raw": 0.051317252449999275
    },
    "views": {
     "Balanced 33:33:33": 72.96187960528026,
     "Emphasis on Accuracy 60:20:20": 70.51933253922925,
     "Emphasis on Cost 20:20:60": 69.46608174549742,
     "Emphasis on Speed 20:60:20": 79.73630555208705,
     "Intelligence only": 67.14748494108278
    }
   },
   "rank": 1,
   "ranked": true,
   "ranks": {
    "A": 1,
    "B": 3,
    "C": 1,
    "capability": 48
   },
   "repo": "https://huggingface.co/torchcast-ai/torchcast-decision-4b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 80.49078635844445,
    "B": 77.41409737698677,
    "C": 80.49078635844445
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21713105699998322,
    "p50_s_raw": 0.033565528499991615,
    "p95_s_adjusted": 0.26205167260007445,
    "p95_s_raw": 0.05602583630003721
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "004acea2e23056bc79415cfa69b9f3ea4254e6d0 paid evaluation (public). Qwen3.5-4B fine-tune served by the author's own torchcast_decision package, measured as shipped: the author's server unchanged at tag v1.3.0 with its bundled decision_config.json (repository torchcast-ai/torchcast-decision-4b at the pinned commit above). Same runtime as the v1.2 row: official pytorch/pytorch:2.9.1-cuda12.8-cudnn9-devel image (digest-pinned) plus an offline hash-locked overlay of the documented packages (transformers 5.8.1, flash-linear-attention 0.5.2, causal-conv1d 1.7.0), on Python 3.11 instead of the documented 3.12. decision_config.json sets attention_mode noncausal_full_attention (the full-attention layers see the whole prompt in both directions, as trained; causal runtimes such as vLLM give different probabilities) and a disclosed yes/no commit floor of 0.801: a yes/no probability with max(P, 1-P) below 0.801 is moved to 0.801 on its own side (answer unchanged; same mechanism as h2o-lightning-4b, v1.1 and v1.2). New in v1.3: new fine-tuned weights; calibration temperature 0.9 (v1.2 used 1.0); CUDA graphs in the non-causal mode (speed only: graph and eager readouts were bit-identical in our check). Prompts over 16,384 tokens keep the head and the tail of the state (documented). Output tokens are 0 (no generation). All 1,500 items answered, none refused; the 15 states longer than 16,384 tokens are cut by the model to head and tail and scored as answered, as for earlier truncated rows. No decision took more than 10 seconds (maximum 7.73 s). The cost is a labelled estimate on the Qwen3.5-4B base-model reference, not a price the author or a host charges. Self-hosted speed adjustment as for every self-hosted row; the device class (A100 SXM4 80GB) is the same as for v1.2. v1.3 is a new row next to v1.2; v1.2 remains listed as its own version.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 87.74496676245083,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 63.2349002873525
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 87.74496676245083
      },
      "calibration_gen": {
       "accuracy": 0.4752475247524752,
       "cc": 25.445755345604557,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": 60.00000000000001,
        "score": -4.129329933493864
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 25.445755345604557
      },
      "composite_gen": {
       "accuracy": 0.900990099009901,
       "cc": 82.96222558951331,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 62.79814921659776
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 82.96222558951331
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.57910175574943,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 90.7373052672483
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.57910175574943
      },
      "jaggedness": {
       "accuracy": 0.7166666666666667,
       "cc": 56.95838990018219,
       "cc_by_type": {
        "choice": 63.90760346487007,
        "noul": 56.75675675675676,
        "score": 50.210809478919735
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 56.95838990018219
      },
      "llm_router": {
       "accuracy": 0.8753993610223643,
       "cc": 76.73558744630976,
       "cc_by_type": {
        "choice": 88.1130580258871,
        "noul": 87.62886597938144,
        "score": 54.46483833366074
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 76.73558744630976
      },
      "long_state": {
       "accuracy": 0.4,
       "cc": -3.1899473094131046,
       "cc_by_type": {
        "choice": 22.58064516129032,
        "noul": 0,
        "score": -32.15048708952963
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 34.50350246279871,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": 25,
        "score": 40.36617749148892
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 34.50350246279871
      },
      "multilingual": {
       "accuracy": 0.8,
       "cc": 68.80518324507379,
       "cc_by_type": {
        "choice": 75.54613628953375,
        "noul": 72.22222222222223,
        "score": 58.64719122346542
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 68.80518324507379
      },
      "negation_pair_gen": {
       "accuracy": 0.8372093023255814,
       "cc": 61.92028089814509,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 66.66666666666667,
        "score": 25.760842694435272
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 61.92028089814509
      },
      "option_order": {
       "accuracy": 0.875,
       "cc": 84.02130492676432,
       "cc_by_type": {
        "choice": 84.02130492676432
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 84.02130492676432
      },
      "perturb_pair": {
       "accuracy": 0.859375,
       "cc": 83.54813299356996,
       "cc_by_type": {
        "choice": 78.97897897897899,
        "noul": 100,
        "score": 71.66542000173091
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 83.54813299356996
      },
      "usecase": {
       "accuracy": 0.7181818181818181,
       "cc": 59.913445939313554,
       "cc_by_type": {
        "choice": 71.81372549019609,
        "noul": 58.33333333333333,
        "score": 49.59327899441123
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 59.913445939313554
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5625,
       "cc": 27.179843956497706,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50,
        "score": 7.730008059969307
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 27.179843956497706
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.9230769230769231,
       "cc": 76.52840979397297,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 29.5852293819189
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 76.52840979397297
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 67.8305005650512,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 50,
        "score": 85.34990877479962
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 67.8305005650512
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 65.5464196197277,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 33.33333333333333,
        "score": 96.82547859847546
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 65.5464196197277
      },
      "en": {
       "accuracy": 0.7836663770634231,
       "cc": 65.85640801672774,
       "cc_by_type": {
        "choice": 75.83627243992628,
        "noul": 72.66187050359711,
        "score": 49.071081106659854
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 65.85640801672774
      },
      "es": {
       "accuracy": 0.9032258064516129,
       "cc": 71.49641471093192,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 14.489244132795754
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 71.49641471093192
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 52.50995210855385,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 89.77029348413149
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 52.50995210855385
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 87.62061012451325,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 62.86183037353974
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 87.62061012451325
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 69.5539278240772,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 75,
        "score": 50.421560008544475
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 69.5539278240772
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 1,
       "cc": 82.31194702287515,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 46.93584106862545
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 82.31194702287515
      },
      "ja": {
       "accuracy": 0.76,
       "cc": 61.54082807954149,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 100,
        "score": 27.51366771757603
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 61.54082807954149
      },
      "ko": {
       "accuracy": 0.8,
       "cc": 78.84328923232499,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 36.52986769697496
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 78.84328923232499
      },
      "mixed": {
       "accuracy": 1,
       "cc": 98.48797537075977,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.4639261122793
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 98.48797537075977
      },
      "nl": {
       "accuracy": 0.7,
       "cc": 55.05285323515513,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": 33.33333333333333,
        "score": 57.79925234615803
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 55.05285323515513
      },
      "no": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 100
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 70.86230835382561,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 60.00000000000001,
        "score": 67.82179598826481
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 70.86230835382561
      },
      "pt": {
       "accuracy": 0.7857142857142857,
       "cc": 75.384975232108,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 57.97578603810361
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 75.384975232108
      },
      "sv": {
       "accuracy": 0.8181818181818182,
       "cc": 70.33653784602723,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 76.54152843169868
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 70.33653784602723
      },
      "tr": {
       "accuracy": 0.5384615384615384,
       "cc": 36.94476888960471,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50,
        "score": 26.051697973161936
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 36.94476888960471
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 64.78031434979971,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 26.599007565528154
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 64.78031434979971
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 32.665765560240864,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": 15.511420974507917
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 32.665765560240864
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.812,
       "cc": 75.7171013605312,
       "cc_by_type": {
        "choice": 75.7171013605312
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 75.7171013605312
      },
      "noul": {
       "accuracy": 0.8613333333333333,
       "cc": 72.26666666666665,
       "cc_by_type": {
        "noul": 72.26666666666665
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 72.26666666666665
      },
      "score": {
       "accuracy": 0.6586666666666666,
       "cc": 50.36312555406744,
       "cc_by_type": {
        "score": 50.36312555406744
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 50.36312555406744
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.7272727272727273,
       "cc": 52.14026099443436,
       "cc_by_type": {
        "choice": 68,
        "noul": 50,
        "score": 38.420782983303056
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 52.14026099443436
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 24.324497869768763,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 33.33333333333333,
        "score": 14.174321766656195
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 24.324497869768763
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 44.95963108067105,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 33.65938104689121
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 44.95963108067105
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 23.877798360690733,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": 8.475500345230103
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 23.877798360690733
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8235294117647058,
       "cc": 40.31542722571074,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": -33.333333333333336,
        "score": 68.77236863365397
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 40.31542722571074
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 40.31629620404986,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 100,
        "score": -53.89387868344788
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 40.31629620404986
      },
      "financial_crime": {
       "accuracy": 0.7,
       "cc": 57.58634953553787,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 42.85714285714286,
        "score": 29.901905749470746
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 57.58634953553787
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 49.580198070423315,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 50,
        "score": 40.55244090116542
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 49.580198070423315
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 62.75826825070436,
       "cc_by_type": {
        "choice": 58.33333333333333,
        "noul": 60.00000000000001,
        "score": 69.94147141877976
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 62.75826825070436
      },
      "knowledge_graphs": {
       "accuracy": 0.6363636363636364,
       "cc": 52.16925909140334,
       "cc_by_type": {
        "choice": 61.4561027837259,
        "noul": 66.66666666666667,
        "score": 28.385007823817443
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 52.16925909140334
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 67.95887685307294,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 37.72675975818526
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.95887685307294
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 10.405704752945985,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 26.916038990020752
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 10.405704752945985
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 59.1188646790882,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 57.356594037264564
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 59.1188646790882
      },
      "model_routing": {
       "accuracy": 0.8709677419354839,
       "cc": 74.91323371602249,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 87.87878787878789,
        "score": 42.70506911343546
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.91323371602249
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 55.12660241169061,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 88.41243679361308
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 55.12660241169061
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 82.28589319945957,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 100,
        "score": 63.57077152038424
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 82.28589319945957
      },
      "risk_assessment": {
       "accuracy": 0.7777777777777778,
       "cc": 61.39301033055477,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 50,
        "score": 48.95735611481702
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 61.39301033055477
      },
      "scientific_discovery": {
       "accuracy": 0.7391304347826086,
       "cc": 58.56002850862949,
       "cc_by_type": {
        "choice": 71.01449275362319,
        "noul": 66.66666666666667,
        "score": 37.9989261055986
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 58.56002850862949
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 97.56474335342972,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 92.69423006028917
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 97.56474335342972
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.80049527648649,
      88.86935442791973
     ],
     "I": [
      61.118891454923116,
      70.91801982503955
     ],
     "capability": [
      72.88106875800261,
      79.21481156880128
     ],
     "composite": null,
     "seed": 82
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.84716845590987,
     "capability": 77.49732669849632,
     "intelligence": 67.14748494108278
    },
    "per_type": {
     "choice": {
      "accuracy": 0.812,
      "calibration": 87.34945811922587,
      "cc": 75.7171013605312,
      "cc_by_type": {
       "choice": 75.7171013605312
      },
      "cc_tiered": 77.32477924973867,
      "cc_tiered_open": 81.67361775115984,
      "cc_tiered_sealed": 72.9759407483175,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.7171013605312
     },
     "noul": {
      "accuracy": 0.8613333333333333,
      "calibration": 87.51851764527558,
      "cc": 72.26666666666665,
      "cc_by_type": {
       "noul": 72.26666666666665
      },
      "cc_tiered": 75.49601861585796,
      "cc_tiered_open": 80.09316770186336,
      "cc_tiered_sealed": 70.89886952985255,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 72.26666666666665
     },
     "score": {
      "accuracy": 0.6586666666666666,
      "calibration": 88.67352960322816,
      "cc": 50.36312555406744,
      "cc_by_type": {
       "score": 50.36312555406744
      },
      "cc_tiered": 48.621656957651716,
      "cc_tiered_open": 47.45302091447617,
      "cc_tiered_sealed": 49.79029300082726,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.36312555406744
     }
    },
    "raw_on_A": {
     "calibration": 82.20877716936431,
     "capability": 74.21943016819003,
     "intelligence": 66.23008316701576
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 78.30501449688185,
    "Emphasis on Accuracy 60:20:20": 73.42477736347463,
    "Emphasis on Cost 20:20:60": 78.71980581674316,
    "Emphasis on Speed 20:60:20": 83.40938493212212,
    "Intelligence only": 67.14748494108278
   }
  },
  {
   "addendum": null,
   "api_flag": true,
   "api_subset": false,
   "author": "MachineFi",
   "axes": {
    "calibration": 85.64268754215436,
    "cost": null,
    "intelligence": 28.409112519894506,
    "speed": 76.18058448046628
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07299797297297295,
      "ece_score": 85.40040540540541,
      "mean_tvd": null,
      "n": 674,
      "n_ece_hard": 296,
      "n_tvd": 0,
      "score": 85.40040540540541,
      "score_before_time_limit": 85.40040540540541,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.3907103825136612,
      "brier": 0.13999579877049192,
      "ece": 0.09047732240437158,
      "n": 366,
      "score": 81.90453551912569,
      "score_before_time_limit": 81.90453551912569,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9413407821229051,
      "interval90_width": 0.513268156424581,
      "n": 358,
      "nrps": 0.10858002406191804,
      "score": 89.62312170193202,
      "score_before_time_limit": 89.62312170193202,
      "time_limit_failed_share": 0,
      "top_ece": 0.04947877094972075
     }
    },
    "score": 85.64268754215436
   },
   "capability": 57.025900031024435,
   "composite_ci95": {
    "A": null,
    "B": null,
    "C": null
   },
   "cost": {
    "basis": "UNPRICED under the frozen pricing_v15 path (sha256 dcfcecb06525ce4a5ece37400ea261b97d2fe5d41155f3cd02b03cac86572835, the module in force since v1.6.21; also under its predecessor 32117d754104b3ead8bebdc9da5f8e0116a7550f4316cfc6874fbe6234c2d72b, against which it was first checked): MachineFi publishes a usage tariff for Trio-Spark of USD 0.042 per 1M billed input tokens with no output charge (docs/visual.md and the README of machinefi/trio-spark), but `trio-spark-v1.3` is in neither AGE_QUALIFIED_API nor LAUNCH_LIST_PRICE_API, and the system declares no base model and releases no weights, so there is no base-model market reference to fall back on. The frozen path therefore returns no eligible price (the same outcome as sage-1.4.0, and as subq-snap-preview before the pricing update).",
    "common_basis": null,
    "kind": "unpriced",
    "usage_estimated_rows": 0,
    "usd_per_1000": null
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Trio-Spark v1.3 (MachineFi)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-15",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Vendor-hosted `platform.machinefi.com/v1/systemone` API, model `trio-spark-v1.3`; latency measured from Sandy (Germany) without API adjustment — the submitter explicitly asked for it to be timed as a hosted endpoint with the network included (over all 1,398 answered requests p50 1.346 s, p95 1.855 s wall; the Speed axis uses the official per-item latency subset; loadavg samples for the whole run are in the measurement receipts on file). Full S1200+P300 draw of `v1.6-regular-20261012` through the unchanged `run_api_full_v161.py` (sha256 `eceea17e…`) and the unchanged `jevbench.adapters.typesafe` (sha256 `780f8794…`); the full-set exposure gate logged all 1,200 sealed items to provider `machinefi` before the first request (this draw's sealed items were already past the two-provider retirement threshold, so the admission cost the reserve nothing). Native System One answer fields on all three question types, nothing generated — with one honesty note the independent review asked for (P2-3): MachineFi's public docs describe only `choice`, so `noul` and `score` are documented here by observation, not by their documentation. The adapter's `noul` → P(yes) orientation is not assumed either: on the 231 PUBLIC items this mapping scores 80.6 % (54 of 67 noul items) against 19.4 % if it were inverted, which settles the direction without touching a sealed item (`output_tokens` 0 on every answered row, 573,254 input tokens over the full set). Hosting implementation and deployed weights were not independently inspected; no weights or training code are published. Regular-queue request `bh-submit:216579c8` (ref BBAEFC3A), measured with the submitter's own API key. 102 of 1,500 requests returned HTTP 422 `request_refused`, for three reasons given in MachineFi's own error messages: 58 because the question has 9 options and Trio-Spark supports at most 8 (\"Question decision has 9 options; Trio-Spark supports at most 8.\" - every 9-option item of this draw), 43 with \"Input exceeds the supported token limit.\" (MachineFi's documented cap of 1,024 model input tokens; answered prompts reach at most 955 billed input tokens), and 1 with \"The decision input cannot be evaluated.\" By type: 76 choice, 17 score, 9 noul. The runner never re-sends a failed item and no request was truncated or shortened, so all 102 are scored as wrong in the full 1,500 denominator. No further requests failed for other reasons. The refusal count was pre-declared at 27–193 before the run, expecting token-limit refusals only; the 8-option limit was not foreseen there, and the add-requests hand-over attributed all 102 to the token cap, which the raw's own error messages correct. The pre-data text is §4 of the pre-registration at sha256 `e4340cdb…`, recorded before dispatch at 22:45Z and 22:55Z — before the dispatch began at 23:03:30Z — and preserved byte-exactly in the measurement receipts on file; the file itself was amended twice afterwards and now hashes `2fbc0ceb…`.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 34.5165059024833,
    "I_sealed": 22.30171913730571,
    "base": 28.409112519894506,
    "excess": 2.3752599948218815,
    "gap": 12.214786765177593,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 48.51140713192591,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 45.513513513513516,
       "hard": 45.265806857502355,
       "judge": 44.62755438365195,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 23.29882677708765,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 21.739130434782616,
       "judge": 0,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 31.739283798436347,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.26121212121212,
       "hard": 24.1500925925926,
       "judge": 29.718813559322033,
       "standard": 39.187407407407406
      }
     },
     "sealed|choice": {
      "cc": 50.113978310534534,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 52.350698856416784,
       "hard": 48.99965305886434,
       "judge": 52.39960822722822,
       "standard": 47.79582366589328
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 27.586206896551737,
       "hard": -12.727272727272732,
       "judge": -16.92307692307692,
       "standard": 25.373134328358216
      }
     },
     "sealed|score": {
      "cc": 16.791179101382603,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 39.64219298245615,
       "hard": -3.60504225352114,
       "judge": 27.741770833333312,
       "standard": 29.732227272727265
      }
     }
    }
   },
   "jevbench_score": null,
   "key": "trio-spark-v1.3",
   "last_measured_on": "2026-10-15",
   "licence": "Proprietary hosted API (no weights or training code released; the repo's own client code is Apache-2.0)",
   "listing": "listed",
   "measured_in": "v1.6.22",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "hosted-api:platform.machinefi.com/v1/systemone#trio-spark-v1.3",
   "not_ranked_because": "Measured, but the frozen pricing path gives it no eligible price: MachineFi publishes a usage tariff for Trio-Spark (USD 0.042 per 1M billed input tokens, output not billed), yet `trio-spark-v1.3` is in neither frozen tariff set (`AGE_QUALIFIED_API`, `LAUNCH_LIST_PRICE_API`) and the system declares no base model and releases no weights, so there is no base-model market reference either — the frozen module (unchanged on this point since the 32117d75... module it was first checked against) returns \"no eligible 30-day list price or base-model estimate\" (verified by executing it; measurement receipts on file). Without a cost axis there is no composite, so the row shows Intelligence, Calibration and Capability and is not ranked, the same outcome as `sage-1.4.0` and as `subq-snap-preview` before the pricing update. The pricing question is open and can be decided in a later release without re-measuring anything.",
   "noul_decisive": {
    "acc_among_decisive": 0.905829596412556,
    "decisive": 223,
    "decisive_rate": 0.6092896174863388,
    "n": 375,
    "supported": true,
    "valid": 366
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 85.64268754215436,
     "cost": null,
     "intelligence": 28.409112519894506,
     "speed": 76.46737050345754
    },
    "board_release": "v1.6.43",
    "capability": 57.025900031024435,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": null,
     "B": null,
     "C": null
    },
    "jevbench_score": null,
    "measured_in": "v1.6.22",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": {
     "A": null,
     "B": null,
     "C": null,
     "capability": null
    },
    "scores": {
     "A": null,
     "B": null,
     "C": null
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 1.2837795019731857,
     "p50_s_raw": 1.2837795019731857,
     "p95_s_adjusted": 1.7570030387170845,
     "p95_s_raw": 1.7570030387170845
    },
    "views": {
     "Balanced 33:33:33": null,
     "Emphasis on Accuracy 60:20:20": null,
     "Emphasis on Cost 20:20:60": null,
     "Emphasis on Speed 20:60:20": null,
     "Intelligence only": null
    }
   },
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://github.com/machinefi/trio-spark",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": null,
    "B": null,
    "C": null
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.3097197005117778,
    "p50_s_raw": 1.3097197005117778,
    "p95_s_adjusted": 1.8397685555450152,
    "p95_s_raw": 1.8397685555450152
   },
   "status": {
    "answered_ok": 1398,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 52.81333333333333,
       "cc_by_type": {
        "choice": 50,
        "noul": 100,
        "score": 8.440000000000003
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 52.81333333333333
      },
      "calibration_gen": {
       "accuracy": 0.5247524752475248,
       "cc": 27.477967258748688,
       "cc_by_type": {
        "choice": 36.37327677624603,
        "noul": 28.000000000000004,
        "score": 18.06062500000003
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 27.477967258748688
      },
      "composite_gen": {
       "accuracy": 0.6138613861386139,
       "cc": 25.287234498413355,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": -18.518518518518523,
        "score": 10.640384615384601
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 25.287234498413355
      },
      "fairness": {
       "accuracy": 0.8125,
       "cc": 72.18962962962964,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 56.56888888888889
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 72.18962962962964
      },
      "jaggedness": {
       "accuracy": 0.4777777777777778,
       "cc": -1.062764445926992,
       "cc_by_type": {
        "choice": 43.69586140519731,
        "noul": -35.13513513513513,
        "score": -11.74901960784316
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.5686900958466453,
       "cc": 45.77148946966644,
       "cc_by_type": {
        "choice": 21.546182970854986,
        "noul": 62.88659793814433,
        "score": 52.8816875
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 45.77148946966644
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -9.75861724758632,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75,
        "score": 19.95095238095238
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.5309090909090909,
       "cc": 13.281860177476112,
       "cc_by_type": {
        "choice": 55.98304532116076,
        "noul": -33.333333333333336,
        "score": 17.195868544600913
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 13.281860177476112
      },
      "negation_pair_gen": {
       "accuracy": 0.6395348837209303,
       "cc": 21.28888888888888,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": -26.66666666666667,
        "score": 7.199999999999984
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 21.28888888888888
      },
      "option_order": {
       "accuracy": 0.8333333333333334,
       "cc": 78.69507323568575,
       "cc_by_type": {
        "choice": 78.69507323568575
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 78.69507323568575
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 70.99001463001464,
       "cc_by_type": {
        "choice": 66.96696696696698,
        "noul": 100,
        "score": 46.003076923076925
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 70.99001463001464
      },
      "usecase": {
       "accuracy": 0.5045454545454545,
       "cc": 21.235606274509806,
       "cc_by_type": {
        "choice": 50.98039215686277,
        "noul": -8.333333333333337,
        "score": 21.059759999999983
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 21.235606274509806
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -25.935775335775343,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": -50,
        "score": -32.56923076923079
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.38461538461538464,
       "cc": -3.255880696364566,
       "cc_by_type": {
        "choice": 17.050691244239633,
        "noul": -33.333333333333336,
        "score": 6.515000000000004
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 57.13679876875722,
       "cc_by_type": {
        "choice": 44.24778761061947,
        "noul": 75,
        "score": 52.16260869565217
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 57.13679876875722
      },
      "el": {
       "accuracy": 0.4,
       "cc": 21.864742396027307,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 99.29999999999998
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.864742396027307
      },
      "en": {
       "accuracy": 0.5760208514335361,
       "cc": 28.572488853859696,
       "cc_by_type": {
        "choice": 50.3545233765758,
        "noul": 15.107913669064743,
        "score": 20.25502951593855
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 28.572488853859696
      },
      "es": {
       "accuracy": 0.6451612903225806,
       "cc": 44.747374532194605,
       "cc_by_type": {
        "choice": 51.06035889070147,
        "noul": 75,
        "score": 8.181764705882344
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 44.747374532194605
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -66.12021857923497,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": -100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 74.80515995872034,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 66.7684210526316
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 74.80515995872034
      },
      "hi": {
       "accuracy": 0.5238095238095238,
       "cc": -20.078258845437613,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": -75,
        "score": -68.475
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.6,
       "cc": 43.96193779460342,
       "cc_by_type": {
        "choice": 43.88152766952456,
        "noul": 100,
        "score": -11.995714285714287
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 43.96193779460342
      },
      "ja": {
       "accuracy": 0.32,
       "cc": -7.710122319301033,
       "cc_by_type": {
        "choice": 14.2176330420969,
        "noul": -40,
        "score": 2.651999999999999
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 12.087619047619038,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 36.26285714285713
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 12.087619047619038
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 71.40999999999998,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 33.33333333333333,
        "score": 97.56333333333333
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 71.40999999999998
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 0.2873246753246785,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 26.83599999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.2873246753246785
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.5384615384615384,
       "cc": 13.017297483490639,
       "cc_by_type": {
        "choice": 61.91282268303006,
        "noul": -40,
        "score": 17.13906976744186
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 13.017297483490639
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 59.82507758790021,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": 32.509999999999984
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 59.82507758790021
      },
      "sv": {
       "accuracy": 0.18181818181818182,
       "cc": -11.35921985815603,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -60,
        "score": 28.049999999999997
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.46153846153846156,
       "cc": -16.836630434782602,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 0,
        "score": -85.29249999999999
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.5714285714285714,
       "cc": 47.08231182795699,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": -26.495
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 47.08231182795699
      },
      "zh": {
       "accuracy": 0.4,
       "cc": 1.6684934086628982,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336,
        "score": 14.609999999999989
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 1.6684934086628982
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.608,
       "cc": 49.36757304961825,
       "cc_by_type": {
        "choice": 49.36757304961825
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 49.36757304961825
      },
      "noul": {
       "accuracy": 0.5386666666666666,
       "cc": 7.733333333333325,
       "cc_by_type": {
        "noul": 7.733333333333325
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 7.733333333333325
      },
      "score": {
       "accuracy": 0.5013333333333333,
       "cc": 20.112120938628173,
       "cc_by_type": {
        "score": 20.112120938628173
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 20.112120938628173
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -14.063725490196083,
       "cc_by_type": {
        "choice": 68,
        "noul": -100,
        "score": -10.191176470588248
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 11.755778078604175,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": -1.309615384615359
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 11.755778078604175
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 51.84317073170731,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 54.30999999999997
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 51.84317073170731
      },
      "demand_forecasting": {
       "accuracy": 0.17647058823529413,
       "cc": -33.242417153996115,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100,
        "score": -6.160000000000032
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 45.9877324879227,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 48.10812499999999
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 45.9877324879227
      },
      "feature_extraction": {
       "accuracy": 0.3157894736842105,
       "cc": -60.33445492662474,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -66.66666666666667,
        "score": -138.86499999999998
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4666666666666667,
       "cc": 14.631414924944329,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": -42.85714285714286,
        "score": 6.35923076923075
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 14.631414924944329
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -17.870361561379593,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -100,
        "score": 30.012608695652165
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.13333333333333333,
       "cc": -25.707962962962966,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": -60,
        "score": -12.95722222222222
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 21.857077304090158,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 33.33333333333333,
        "score": -3.5222727272727017
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 21.857077304090158
      },
      "lead_generation": {
       "accuracy": 0.48,
       "cc": 5.086647874089732,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": -33.333333333333336,
        "score": -28.150909090909092
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 5.086647874089732
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -38.41625896057349,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": -85.14125000000001
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 18.304845738252638,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -19.999999999999996,
        "score": 32.496111111111084
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 18.304845738252638
      },
      "model_routing": {
       "accuracy": 0.5345622119815668,
       "cc": 43.44288566468256,
       "cc_by_type": {
        "choice": 8.831168831168855,
        "noul": 78.78787878787878,
        "score": 42.709609375000014
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 43.44288566468256
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": -0.4494520341725945,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 33.20058823529411
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 54.2654650858272,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 46.176153846153845
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.2654650858272
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": -38.847736887858595,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -100,
        "score": -72.20823529411766
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.34782608695652173,
       "cc": -14.521700483091797,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -100,
        "score": 28.89866666666664
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.7916666666666666,
       "cc": 67.4370359617906,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 50,
        "score": 73.66342105263158
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 67.4370359617906
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.49196128193122,
      86.51567270201278
     ],
     "I": [
      22.829086556647436,
      32.828914916616085
     ],
     "capability": [
      52.19692696654943,
      58.86725786769748
     ],
     "composite": {
      "A": null,
      "B": null,
      "C": null
     },
     "seed": 78
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 9.839526770355711,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.64268754215436,
     "capability": 57.025900031024435,
     "intelligence": 28.409112519894506
    },
    "per_type": {
     "choice": {
      "accuracy": 0.608,
      "calibration": 85.40040540540541,
      "cc": 49.36757304961825,
      "cc_by_type": {
       "choice": 49.36757304961825
      },
      "cc_tiered": 49.31269272123022,
      "cc_tiered_open": 48.51140713192591,
      "cc_tiered_sealed": 50.113978310534534,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.36757304961825
     },
     "noul": {
      "accuracy": 0.5386666666666666,
      "calibration": 81.90453551912569,
      "cc": 7.733333333333325,
      "cc_by_type": {
       "noul": 7.733333333333325
      },
      "cc_tiered": 11.649413388543826,
      "cc_tiered_open": 23.29882677708765,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 7.733333333333325
     },
     "score": {
      "accuracy": 0.5013333333333333,
      "calibration": 89.62312170193202,
      "cc": 20.112120938628173,
      "cc_by_type": {
       "score": 20.112120938628173
      },
      "cc_tiered": 24.265231449909475,
      "cc_tiered_open": 31.739283798436347,
      "cc_tiered_sealed": 16.791179101382603,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 20.112120938628173
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.10133333333333339,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.02400000000000002,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.04533333333333334,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": null,
    "Emphasis on Accuracy 60:20:20": null,
    "Emphasis on Cost 20:20:60": null,
    "Emphasis on Speed 20:60:20": null,
    "Intelligence only": null
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Turion AI (Hugging Face)",
   "axes": {
    "calibration": 90.90010349317609,
    "cost": 46.586791013615496,
    "intelligence": 70.85736825150448,
    "speed": 85.85779594425273
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06226496065573764,
      "ece_score": 87.54700786885248,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 87.54700786885248,
      "score_before_time_limit": 87.54700786885248,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.13136729222520105,
      "brier": 0.06171283997572119,
      "ece": 0.03415193029490614,
      "n": 373,
      "score": 93.16961394101877,
      "score_before_time_limit": 93.16961394101877,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.989247311827957,
      "interval90_width": 0.2961469534050179,
      "n": 372,
      "nrps": 0.04280226424126749,
      "score": 91.98368866965704,
      "score_before_time_limit": 91.98368866965704,
      "time_limit_failed_share": 0,
      "top_ece": 0.05876198118279579
     }
    },
    "score": 90.90010349317609
   },
   "capability": 80.87873587234029,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense gemma, 31B active parameters; throughput 10664.9184474 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=native). The row's own scored cost basis: 1,580,092 input + 0 output tokens over 1,485 decisions = 1064.03501684 tokens/decision. GPU-time cost USD 0.0603170787128884/1,000; old list-price cost USD 0.0957631515151515/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0603170787128884/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.09480552
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.06031707871288843
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Turion Jev 31B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.1",
    "results_sha256": "d09d19d9cdcbda6a463c05f286439044881ccde61119b659b5f4d980f9b237e9",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Author's turion_jev server on evaluator-rented Verda RTX PRO 6000 Blackwell 96 GB, Finland, EU, 24 October 2026 19:12-19:31 UTC. Pinned local weights and code, isolated container, serial native requests. Shred and exact-instance teardown receipts retained. Approximately USD 0.83 rental is separate from the estimated inference tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 71.68391080858707,
    "I_sealed": 70.0308256944219,
    "base": 70.85736825150448,
    "excess": -8.186441656190546,
    "gap": 1.6530851141651652,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 79.73511307776322,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 79.2387543252595,
       "judge": 80.22412656558998,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 69.46997929606626,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 73.91304347826086,
       "judge": 60.00000000000001,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 65.84664005193176,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 78.03246969696971,
       "hard": 78.83088518518518,
       "judge": 59.825742372881365,
       "standard": 42.8165814814815
      }
     },
     "sealed|choice": {
      "cc": 84.4786228933838,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 94.28208386277001,
       "hard": 82.30600208164681,
       "judge": 83.2517140058766,
       "standard": 85.76249736342545
      }
     },
     "sealed|noul": {
      "cc": 61.4822943397313,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 68.96551724137932,
       "hard": 52.727272727272734,
       "judge": 56.92307692307692,
       "standard": 82.08955223880596
      }
     },
     "sealed|score": {
      "cc": 64.13155985015064,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 85.21135614035087,
       "hard": 61.55521323943661,
       "judge": 60.4002734375,
       "standard": 64.34128454545454
      }
     }
    }
   },
   "jevbench_score": 59.63832789350321,
   "key": "turion-jev-31b",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 weights (model card, LICENSE and NOTICE; the LICENSE appendix placeholder is unfilled); author serving code under the same repository licence (Apache-2.0 per the card and the pyproject.toml licence field; the six author modules carry no per-file licence headers); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.1",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "turion-ai/turion-jev-31b@9edd5783ed1735cbf9e3e8ae7a546126ae05b0ce",
   "new_in": "v1.7.1",
   "noul_decisive": {
    "acc_among_decisive": 0.9537037037037037,
    "decisive": 324,
    "decisive_rate": 0.868632707774799,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 111,
   "ranked": true,
   "ranks": {
    "A": 111,
    "B": 108,
    "C": 87,
    "capability": 30
   },
   "repo": "https://huggingface.co/turion-ai/turion-jev-31b",
   "scores": {
    "A": 59.63832789350321,
    "B": 60.00416461737817,
    "C": 59.63832789350321
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3982319030001463,
    "p50_s_raw": 0.12411595150007315,
    "p95_s_adjusted": 0.6517549772998905,
    "p95_s_raw": 0.25087748864994525
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "First measurement of turion-ai/turion-jev-31b at 9edd5783, a Gemma-4-31B-it fine-tune. Author's turion_jev server, in-process vLLM 0.32.0 with the author's official online FP8 engine options; evaluator launcher and score re-key shim. FP8 versus BF16 agreement is unknown. Training data partly comes from a Jev 1.13-distilled stream; contamination was not checked. No LFS weight shards are shared with KnowLine-31B-Gen1 or the base. Apache-2.0 LICENSE appendix is unfilled. Exactly 1485 answers and 15 native HTTP 422 refusals under the author's 65536-token maximum; the refused items count wrong in the full 1500-item denominator. Native usage counts the author's read-twice layout and reports zero output tokens. First 50 responses repeated bit-identically. No evaluator tuning or second measurement was performed.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 89.21263333333333,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 67.63789999999999
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 89.21263333333333
      },
      "calibration_gen": {
       "accuracy": 0.8712871287128713,
       "cc": 75.00771577302051,
       "cc_by_type": {
        "choice": 84.09331919406152,
        "noul": 60.00000000000001,
        "score": 80.929828125
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 75.00771577302051
      },
      "composite_gen": {
       "accuracy": 0.8910891089108911,
       "cc": 84.44667654900053,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 92.59259259259258,
        "score": 70.50353461538461
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 84.44667654900053
      },
      "fairness": {
       "accuracy": 0.9791666666666666,
       "cc": 95.8382283068783,
       "cc_by_type": {
        "choice": 90.47619047619048,
        "noul": 100,
        "score": 97.03849444444444
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 95.8382283068783
      },
      "jaggedness": {
       "accuracy": 0.7888888888888889,
       "cc": 62.240463690594304,
       "cc_by_type": {
        "choice": 76.90086621751685,
        "noul": 40.54054054054055,
        "score": 69.2799843137255
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 62.240463690594304
      },
      "llm_router": {
       "accuracy": 0.9105431309904153,
       "cc": 80.30526687311755,
       "cc_by_type": {
        "choice": 97.62261160517741,
        "noul": 83.50515463917525,
        "score": 59.78803437500001
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 80.30526687311755
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7777777777777778,
       "cc": 61.09357550319097,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 50,
        "score": 70.39412857142857
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 61.09357550319097
      },
      "multilingual": {
       "accuracy": 0.7563636363636363,
       "cc": 60.43629997826323,
       "cc_by_type": {
        "choice": 78.4805999347897,
        "noul": 41.66666666666667,
        "score": 61.161633333333334
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 60.43629997826323
      },
      "negation_pair_gen": {
       "accuracy": 0.9186046511627907,
       "cc": 80.80396888888889,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 73.33333333333334,
        "score": 75.74524
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 80.80396888888889
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.921875,
       "cc": 86.2892036883037,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 70.87962307692308
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 86.2892036883037
      },
      "usecase": {
       "accuracy": 0.7636363636363637,
       "cc": 61.87657109542484,
       "cc_by_type": {
        "choice": 73.03921568627452,
        "noul": 50,
        "score": 62.59049759999999
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 61.87657109542484
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.6875,
       "cc": 60.79450915750915,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100,
        "score": 39.5263846153846
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 60.79450915750915
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 64.84692187403992,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": 100,
        "score": 36.01541999999998
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 64.84692187403992
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 82.94197938950879,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 72.71974347826087
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 82.94197938950879
      },
      "el": {
       "accuracy": 0.8,
       "cc": 72.0993022656735,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 33.33333333333333,
        "score": 99.72435
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 72.0993022656735
      },
      "en": {
       "accuracy": 0.8583840139009556,
       "cc": 74.61181983502063,
       "cc_by_type": {
        "choice": 85.72143371450188,
        "noul": 69.78417266187051,
        "score": 68.32985312868949
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 74.61181983502063
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 72.19439843585069,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 26.371123529411776
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 72.19439843585069
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 65.66059380692168,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 96.43533333333335
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.66059380692168
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 86.48357647058823,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 80.6272
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 86.48357647058823
      },
      "hi": {
       "accuracy": 0.8571428571428571,
       "cc": 70.84362499999999,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 50,
        "score": 62.53087499999999
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 70.84362499999999
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 67.81848444865085,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 17.48507142857143
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 67.81848444865085
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 54.2110062801165,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 60.00000000000001,
        "score": 16.930080000000004
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 54.2110062801165
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 23.18293571428571,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 69.54880714285711
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 23.18293571428571
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 75.42267222222222,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 92.93468333333334
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 75.42267222222222
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 5.832351341991341,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 43.47107999999999
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 5.832351341991341
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 62.397199224806194,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 67.19159767441859
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 62.397199224806194
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 50.45449340075149,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 19.999999999999996,
        "score": 52.577387096774196
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 50.45449340075149
      },
      "sv": {
       "accuracy": 0.5454545454545454,
       "cc": 35.42225673758865,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 19.999999999999996,
        "score": 37.33059999999999
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 35.42225673758865
      },
      "tr": {
       "accuracy": 0.7692307692307693,
       "cc": 75.28016902173913,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 47.579637500000004
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 75.28016902173913
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 79.23511182795698,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 69.9634
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 79.23511182795698
      },
      "zh": {
       "accuracy": 0.9,
       "cc": 74.1384,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 22.415200000000013
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 74.1384
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8786666666666667,
       "cc": 84.32805832488184,
       "cc_by_type": {
        "choice": 84.32805832488184
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 84.32805832488184
      },
      "noul": {
       "accuracy": 0.824,
       "cc": 64.8,
       "cc_by_type": {
        "noul": 64.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 64.8
      },
      "score": {
       "accuracy": 0.7706666666666667,
       "cc": 65.96814440433214,
       "cc_by_type": {
        "score": 65.96814440433214
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 65.96814440433214
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 32.16686274509803,
       "cc_by_type": {
        "choice": 68,
        "noul": 0,
        "score": 28.50058823529409
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 32.16686274509803
      },
      "code_linting": {
       "accuracy": 0.9583333333333334,
       "cc": 89.16438333333333,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 67.49314999999999
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 89.16438333333333
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 51.56341246612467,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 50,
        "score": 20.950400000000002
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 51.56341246612467
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -21.93144417641325,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": 16.07701250000001
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6470588235294118,
       "cc": 49.43295836352657,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": 33.33333333333333,
        "score": 72.93655625
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 49.43295836352657
      },
      "feature_extraction": {
       "accuracy": 0.631578947368421,
       "cc": 15.853061635220136,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 66.66666666666667,
        "score": -56.214399999999976
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 15.853061635220136
      },
      "financial_crime": {
       "accuracy": 0.6666666666666666,
       "cc": 42.12228292034762,
       "cc_by_type": {
        "choice": 90.19607843137256,
        "noul": -14.28571428571429,
        "score": 50.4564846153846
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 42.12228292034762
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 64.42278787052466,
       "cc_by_type": {
        "choice": 79.09407665505228,
        "noul": 50,
        "score": 64.17428695652173
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 64.42278787052466
      },
      "insurance_claims": {
       "accuracy": 0.4666666666666667,
       "cc": 34.4684962962963,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 45.90548888888889
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 34.4684962962963
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 84.01493193173707,
       "cc_by_type": {
        "choice": 74.30406852248395,
        "noul": 100,
        "score": 77.74072727272727
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 84.01493193173707
      },
      "lead_generation": {
       "accuracy": 0.84,
       "cc": 72.71454107744108,
       "cc_by_type": {
        "choice": 100,
        "noul": 77.77777777777777,
        "score": 40.365845454545465
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 72.71454107744108
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 39.85034726702509,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 50.7338375
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 39.85034726702509
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 60.605612962962965,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 61.81683888888888
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 60.605612962962965
      },
      "model_routing": {
       "accuracy": 0.8709677419354839,
       "cc": 75.78775654310967,
       "cc_by_type": {
        "choice": 94.15584415584415,
        "noul": 84.84848484848484,
        "score": 48.35894062500002
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 75.78775654310967
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 23.279220563772537,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 81.35397647058824
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 23.279220563772537
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 55.536463932338165,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 66.70224230769233
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 55.536463932338165
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 19.636584680768852,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0,
        "score": 3.2447294117646908
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 19.636584680768852
      },
      "scientific_discovery": {
       "accuracy": 0.4782608695652174,
       "cc": 14.966232753623189,
       "cc_by_type": {
        "choice": 13.04347826086957,
        "noul": -33.333333333333336,
        "score": 65.18855333333333
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 14.966232753623189
      },
      "search_retrieval": {
       "accuracy": 0.9583333333333334,
       "cc": 95.69731359493039,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 97.76809736842105
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 95.69731359493039
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.9290223314672,
      90.95948441960888
     ],
     "I": [
      65.86419487775888,
      74.63809657795153
     ],
     "capability": [
      76.9773704992769,
      82.12352375945959
     ],
     "composite": null,
     "seed": 79
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.90010349317609,
     "capability": 80.87873587234029,
     "intelligence": 70.85736825150448
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8786666666666667,
      "calibration": 87.54700786885248,
      "cc": 84.32805832488184,
      "cc_by_type": {
       "choice": 84.32805832488184
      },
      "cc_tiered": 82.1068679855735,
      "cc_tiered_open": 79.73511307776322,
      "cc_tiered_sealed": 84.4786228933838,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 84.32805832488184
     },
     "noul": {
      "accuracy": 0.824,
      "calibration": 93.16961394101877,
      "cc": 64.8,
      "cc_by_type": {
       "noul": 64.8
      },
      "cc_tiered": 65.47613681789878,
      "cc_tiered_open": 69.46997929606626,
      "cc_tiered_sealed": 61.4822943397313,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 64.8
     },
     "score": {
      "accuracy": 0.7706666666666667,
      "calibration": 91.98368866965704,
      "cc": 65.96814440433214,
      "cc_by_type": {
       "score": 65.96814440433214
      },
      "cc_tiered": 64.9890999510412,
      "cc_tiered_open": 65.84664005193176,
      "cc_tiered_sealed": 64.13155985015064,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 65.96814440433214
     }
    },
    "raw_on_A": {
     "calibration": 87.8132406756091,
     "capability": 79.27535065782627,
     "intelligence": 70.73746064004342
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 55.14825567448587,
    "Emphasis on Accuracy 60:20:20": 57.52945820722256,
    "Emphasis on Cost 20:20:60": 48.146103616575424,
    "Emphasis on Speed 20:60:20": 61.55244778931542,
    "Intelligence only": 61.51352321971125
   }
  },
  {
   "adapter_id": "images/typecastlm/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mikhail Gribov",
   "axes": {
    "calibration": 83.29951051808085,
    "cost": 78.50093844783655,
    "intelligence": 29.78900656204624,
    "speed": 92.66059621716312
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.14894691278376873,
      "ece_score": 70.21061744324626,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 70.21061744324626,
      "score_before_time_limit": 70.21061744324626,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.42666666666666664,
      "brier": 0.1443460310902931,
      "ece": 0.03743872703084413,
      "n": 375,
      "score": 92.51225459383117,
      "score_before_time_limit": 92.51225459383117,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9973333333333333,
      "interval90_width": 0.8611111111111112,
      "n": 375,
      "nrps": 0.11438335298438909,
      "score": 87.17565951716512,
      "score_before_time_limit": 87.17565951716512,
      "time_limit_failed_share": 0,
      "top_ece": 0.07105172833615442
     }
    },
    "score": 83.29951051808085
   },
   "capability": 56.544258540063545,
   "class": "system-one-open",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 860,659 input + 0 output tokens over 1,624 decisions = 529.962438424 tokens/decision. GPU-time cost USD 0.00520757322122523/1,000; old list-price cost USD 0.0158988731527094/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00520757322122523/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.005207573221225232
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "typecastlm (Mikhail Gribov, Qwen3.5-4B computed head)",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX5090), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX5090",
   "intelligence": {
    "I_open": 34.211044348094305,
    "I_sealed": 25.366968775998174,
    "base": 29.78900656204624,
    "excess": 6.270687929657887,
    "gap": 8.84407557209613,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 53.948175356316256,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 45.26580685750237,
       "judge": 52.537903757415954,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 19.30296756383713,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": 39.13043478260869,
       "judge": 0,
       "standard": 11.111111111111116
      }
     },
     "open|score": {
      "cc": 29.381990124129523,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 59.87815307647144,
       "hard": 32.63833455597018,
       "judge": 28.21824515037382,
       "standard": 9.366837244910798
      }
     },
     "sealed|choice": {
      "cc": 46.35542091610937,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 61.4219200354741,
       "hard": 37.89592760180995,
       "judge": 51.58286778398511,
       "standard": 47.89998768321223
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 19.999999999999996,
       "hard": -22.330097087378643,
       "judge": 2.6315789473684292,
       "standard": 15.492957746478876
      }
     },
     "sealed|score": {
      "cc": 29.745485411885156,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 52.652319160951166,
       "hard": 25.940906558427802,
       "judge": 26.848256959038974,
       "standard": 30.247068923536112
      }
     }
    }
   },
   "jevbench_score": 20.54575277268221,
   "key": "typecastlm",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (package and weights)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9023255813953488,
    "decisive_rate": 0.5733333333333334,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.29951051808085,
     "cost": 63.95900966831593,
     "intelligence": 29.78900656204624,
     "speed": 92.76354061232337
    },
    "board_release": "v1.6.43",
    "capability": 56.544258540063545,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 76,
    "capability_rank": 102,
    "composite_ci95": {
     "A": [
      12.880576558614674,
      26.104438970675695
     ],
     "B": [
      10.666557109719218,
      22.72472546215454
     ],
     "C": [
      8.944844832371304,
      18.12808261852479
     ]
    },
    "jevbench_score": 19.72256430004839,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 98,
    "rank": 133,
    "ranks": {
     "A": 133,
     "B": 142,
     "C": 140,
     "capability": 156
    },
    "scores": {
     "A": 19.72256430004839,
     "B": 16.813084809424783,
     "C": 13.69622520836694
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20903298105113208,
     "p50_s_raw": 0.02951649052556604,
     "p95_s_adjusted": 0.25318100801086985,
     "p95_s_raw": 0.051590504005434926
    },
    "views": {
     "Balanced 33:33:33": 17.752271764169233,
     "Emphasis on Accuracy 60:20:20": 13.960999831173599,
     "Emphasis on Cost 20:20:60": 19.448558330257274,
     "Emphasis on Speed 20:60:20": 21.764369978155518,
     "Intelligence only": 10.57372598609905
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 185,
   "ranked": true,
   "ranks": {
    "A": 185,
    "B": 187,
    "C": 187,
    "capability": 188
   },
   "repo": "https://github.com/mihail-gribov/typecastlm",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 20.54575277268221,
    "B": 17.28539860544098,
    "C": 14.267883869918203
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.21090144504560157,
    "p50_s_raw": 0.03045072252280079,
    "p95_s_adjusted": 0.25695721871452404,
    "p95_s_raw": 0.05347860935726202
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": null,
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.58498026450093,
      85.01640087455884
     ],
     "I": [
      25.093945135273305,
      33.49496683838513
     ],
     "capability": [
      52.490530265893305,
      58.297640238106396
     ],
     "composite": null,
     "seed": 107
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.29951051808085,
     "capability": 56.544258540063545,
     "intelligence": 29.78900656204624
    },
    "per_type": {
     "choice": {
      "accuracy": 0.5933333333333334,
      "calibration": 70.21061744324626,
      "cc": 47.99314153633374,
      "cc_by_type": {
       "choice": 47.99314153633374
      },
      "cc_tiered": 50.15179813621281,
      "cc_tiered_open": 53.948175356316256,
      "cc_tiered_sealed": 46.35542091610937,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 47.99314153633374
     },
     "noul": {
      "accuracy": 0.5173333333333333,
      "calibration": 92.51225459383117,
      "cc": 3.4666666666666623,
      "cc_by_type": {
       "noul": 3.4666666666666623
      },
      "cc_tiered": 9.651483781918564,
      "cc_tiered_open": 19.30296756383713,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 3.4666666666666623
     },
     "score": {
      "accuracy": 0.48533333333333334,
      "calibration": 87.17565951716512,
      "cc": 31.463946158492806,
      "cc_by_type": {
       "score": 31.463946158492806
      },
      "cc_tiered": 29.56373776800734,
      "cc_tiered_open": 29.381990124129523,
      "cc_tiered_sealed": 29.745485411885156,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 31.463946158492806
     }
    },
    "raw_on_A": {
     "calibration": 79.3986887571586,
     "capability": 55.69880635159235,
     "intelligence": 31.998923946026117
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 18.64900264654086,
    "Emphasis on Accuracy 60:20:20": 14.285118615527779,
    "Emphasis on Cost 20:20:60": 21.492143643722315,
    "Emphasis on Speed 20:60:20": 22.555562031247895,
    "Intelligence only": 10.57372598609905
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Mikhail Gribov / @MihailGGribov (mihail.gribov.rs@gmail.com)",
   "axes": {
    "calibration": 77.35692434429644,
    "cost": 79.34178886317649,
    "intelligence": 35.43137209354293,
    "speed": 92.811325470547
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10031579026354083,
      "ece_score": 79.93684194729184,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 79.93684194729184,
      "score_before_time_limit": 79.93684194729184,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.22666666666666668,
      "brier": 0.19319243660768046,
      "ece": 0.16531579253832934,
      "n": 375,
      "score": 66.93684149233412,
      "score_before_time_limit": 66.93684149233412,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9893333333333333,
      "interval90_width": 0.7702222222222224,
      "n": 375,
      "nrps": 0.1175329689730567,
      "score": 85.19708959326337,
      "score_before_time_limit": 85.19708959326337,
      "time_limit_failed_share": 0,
      "top_ece": 0.08926261958083807
     }
    },
    "score": 77.35692434429644
   },
   "capability": 56.39414821891969,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 733,833 input + 0 output tokens over 1,477 decisions = 496.840216655 tokens/decision. GPU-time cost USD 0.00488210412643251/1,000; old list-price cost USD 0.0149052064996615/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00488210412643251/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.029810319999999994
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004882104126432508
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "TypeCastLM 1.4.0 (Mikhail Gribov, Qwen3.5-4B computed head)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. A FOLLOW-UP OF THE PUBLISHED typecastlm ROW: the author asks to 'measure 1.4.0 to replace it' and to list it as 'TypeCastLM'; measured here as a CANDIDATE - reported separately from the earlier serving version. Weights mihailgribov/typecastlm-qwen3.5-3.8b tag v1.4.0 = commit 0984c3e832f7c28495134bc1d22c66bb0e0a555d (GGUF files excluded), package typecastlm[server]==1.2.2 (latest on PyPI) + flash-linear-attention + fla-core, the model card's own install lines, in an isolated venv (torch 2.14.1+cu130, transformers 5.19.0). Served with the author's own `typecastlm-serve --model <local v1.4.0 snapshot> --host 127.0.0.1 --port 8000 --device cuda`; the UNCHANGED typesafe adapter with the published row's own wire model; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. THE 23 LONG ITEMS ARE ANSWERED, folded in the middle to the server's own 32,768-token state context (its documented behaviour; max input 32,953 tokens observed). One RTX 5090 32 GB (Lium, France). Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, credential-free. add-requests run 59, benchmarkheaven.com/submit ref 34CA4AED",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX 5090",
   "intelligence": {
    "I_open": 38.13170692188825,
    "I_sealed": 32.731037265197614,
    "base": 35.43137209354293,
    "excess": 2.8272820142523947,
    "gap": 5.4006696566906385,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 52.552512469264286,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 61.08108108108109,
       "hard": 43.378420887071414,
       "judge": 60.44825313117996,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 25.505175983436857,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 10.000000000000009,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 36.33743231296363,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 64.8527119217414,
       "hard": 32.894905449875765,
       "judge": 45.884441468733286,
       "standard": 14.644332501095958
      }
     },
     "sealed|choice": {
      "cc": 47.566937838880506,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 65.41275589387332,
       "hard": 42.986425339366505,
       "judge": 49.72067039106147,
       "standard": 44.57445498214066
      }
     },
     "sealed|noul": {
      "cc": 19.912391055582344,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 60.00000000000001,
       "hard": -6.796116504854366,
       "judge": 39.47368421052631,
       "standard": 23.943661971830977
      }
     },
     "sealed|score": {
      "cc": 30.71378290112999,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 52.96711799317693,
       "hard": 27.922053490937206,
       "judge": 28.3107755484573,
       "standard": 28.775085204501117
      }
     }
    }
   },
   "jevbench_score": 31.127309270665172,
   "key": "typecastlm-1.4.0",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "mihailgribov/typecastlm-qwen3.5-3.8b@0984c3e832f7c28495134bc1d22c66bb0e0a555d",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8068965517241379,
    "decisive_rate": 0.7733333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 77.35692434429644,
     "cost": 64.79986008365586,
     "intelligence": 35.43137209354293,
     "speed": 92.95077486551261
    },
    "board_release": "v1.6.43",
    "capability": 56.39414821891969,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 78,
    "capability_rank": 104,
    "composite_ci95": {
     "A": [
      19.06853775916415,
      39.61576669827309
     ],
     "B": [
      16.309735285734533,
      35.62882083236784
     ],
     "C": [
      13.242040110530658,
      27.510949096022983
     ]
    },
    "jevbench_score": 29.827386072784904,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 70,
    "rank": 102,
    "ranks": {
     "A": 102,
     "B": 103,
     "C": 108,
     "capability": 159
    },
    "scores": {
     "A": 29.827386072784904,
     "B": 26.272910136782546,
     "C": 20.713462550545067
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21072090499219484,
     "p50_s_raw": 0.030360452496097423,
     "p95_s_adjusted": 0.2405552782700397,
     "p95_s_raw": 0.04527763913501986
    },
    "views": {
     "Balanced 33:33:33": 27.685082169472313,
     "Emphasis on Accuracy 60:20:20": 22.64780945981884,
     "Emphasis on Cost 20:20:60": 29.441987709970327,
     "Emphasis on Speed 20:60:20": 33.066455632229754,
     "Intelligence only": 17.791964524810385
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 162,
   "ranked": true,
   "ranks": {
    "A": 162,
    "B": 165,
    "C": 164,
    "capability": 191
   },
   "repo": "https://huggingface.co/mihailgribov/typecastlm-qwen3.5-3.8b",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 31.127309270665172,
    "B": 27.06950858655119,
    "C": 21.616186993517474
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2110814839892555,
    "p50_s_raw": 0.03054074199462775,
    "p95_s_adjusted": 0.2479803759983042,
    "p95_s_raw": 0.048990187999152106
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen/Qwen3.5-4B",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      71.77140344619215,
      79.65897110167947
     ],
     "I": [
      29.58940031351787,
      39.89308222540656
     ],
     "capability": [
      51.45876220790242,
      59.168420588826265
     ],
     "composite": null,
     "seed": 147
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 77.35692434429644,
     "capability": 56.39414821891969,
     "intelligence": 35.43137209354293
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6026666666666667,
      "calibration": 79.93684194729184,
      "cc": 49.186741566647385,
      "cc_by_type": {
       "choice": 49.186741566647385
      },
      "cc_tiered": 50.059725154072396,
      "cc_tiered_open": 52.552512469264286,
      "cc_tiered_sealed": 47.566937838880506,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 49.186741566647385
     },
     "noul": {
      "accuracy": 0.624,
      "calibration": 66.93684149233412,
      "cc": 24.8,
      "cc_by_type": {
       "noul": 24.8
      },
      "cc_tiered": 22.708783519509602,
      "cc_tiered_open": 25.505175983436857,
      "cc_tiered_sealed": 19.912391055582344,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 24.8
     },
     "score": {
      "accuracy": 0.5466666666666666,
      "calibration": 85.19708959326337,
      "cc": 33.379404446849435,
      "cc_by_type": {
       "score": 33.379404446849435
      },
      "cc_tiered": 33.52560760704681,
      "cc_tiered_open": 36.33743231296363,
      "cc_tiered_sealed": 30.71378290112999,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 33.379404446849435
     }
    },
    "raw_on_A": {
     "calibration": 73.20180288665794,
     "capability": 57.51186004997652,
     "intelligence": 41.821917213295094
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 29.193902855994832,
    "Emphasis on Accuracy 60:20:20": 23.23728040817137,
    "Emphasis on Cost 20:20:60": 32.68831925260945,
    "Emphasis on Speed 20:60:20": 34.323073646110046,
    "Intelligence only": 17.791964524810385
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ezoushen (Hugging Face)",
   "axes": {
    "calibration": 90.73266984958518,
    "cost": 75.39758639297672,
    "intelligence": 58.18796963108992,
    "speed": 79.61785431820445
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04999026270666329,
      "ece_score": 90.00194745866735,
      "mean_tvd": null,
      "n": 746,
      "n_ece_hard": 350,
      "n_tvd": 0,
      "score": 90.00194745866735,
      "score_before_time_limit": 90.00194745866735,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.1737967914438503,
      "brier": 0.06862480143372438,
      "ece": 0.04346641547130901,
      "n": 374,
      "score": 91.3067169057382,
      "score_before_time_limit": 91.3067169057382,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9546666666666667,
      "interval90_width": 0.2602222222222221,
      "n": 375,
      "nrps": 0.059748954499922735,
      "score": 90.88934518434999,
      "score_before_time_limit": 90.88934518434999,
      "time_limit_failed_share": 0,
      "top_ece": 0.06123207090653871
     }
    },
    "score": 90.73266984958518
   },
   "capability": 74.46031974033755,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe qwen, 3B active parameters; throughput 37999.6666667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 618,462 input + 0 output tokens over 1,489 decisions = 415.353928811 tokens/decision. GPU-time cost USD 0.00660815853589348/1,000; old list-price cost USD 0.129798102753526/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00660815853589348/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.12945625
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.006608158535893478
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "Umpire 35B-A3B is a 35B-total, 3B-active mixture-of-experts model (qwen3_5_moe architecture), declared base ornith-ai/Ornith-1.5-35B-A3B, with final-position option-slot logits. AUTHOR CLAIM: these BF16 weights are the author's served 4-bit weights dequantized; tensor parity was not established. Only the BF16 CUDA path was measured; no MLX or other weight variant was measured. SERVING: the author's unchanged Transformers server, BF16, use_cache false, logits_to_keep 1, temperature 1, up to 255 option slots, thinking off, one forward pass, no generated text, truncation, retries or ensemble. The vLLM engine was not started. Contributor code ran only on a disposable rented pod with network cut. KERNEL PATH: flash-linear-attention and causal-conv1d were unavailable; Transformers' reference PyTorch linear-attention path is inferred from server warnings, not kernel tracing; no separate scored-run kernel trace was captured. ITEMS: all 1,500 items were sent; exactly 5 exceeded the native 16,384-token input limit, matching the preregistered count, and returned HTTP 400 context_length_exceeded, scored as misses and kept in the denominator; these carry no usage and are unbilled. 1,495 items were answered. Boundary checks answered 16,384 tokens and refused 16,385, 80,000 and 84,000 tokens with normal recovery. TOKENS: output tokens reported as 0; native input usage was 621,390 tokens on answered requests, identical to first-party offline counts with 0 usage mismatches. A one-option request returns deterministically with zero usage. TOKENIZER: exact prompt bytes, slot ids, encoded ids and usage matched over 543 public and synthetic cases, including 231 public rows. TRAINING: the card claims Open-Jev probability-distribution training and its own public test numbers; these claims were not independently verified. Training data were not inspected; overlap with public or sealed JevBench items is unchecked. DETERMINISM: 49 of 50 replayed requests were bit-identical, with 0 argmax flips (recorded maximum probability difference 0.0). WARM-UP: public pre-warm preceded the scored fresh server's timed requests; separate public smoke recorded about 22 seconds for model loading and 21 seconds for warm-up. LATENCY: measured loopback p50 0.405 s, p95 0.494 s, maximum 0.627 s; no request took 10 seconds or longer. The share view applies the common self-hosted latency assumption, twice measured p50 plus 0.15 s, an assumption rather than a hosted-endpoint measurement. HARDWARE: one A100 80 GB GPU; weights alone occupy 64.6 GiB. No peak-memory measurement is available. LICENCE: weights CC BY-NC 4.0 (non-commercial use only); serving code Apache-2.0, with MIT SemIf notice preserved. One scored run, with every pre-input gate passed before scored input. The 27 Oct 2026 draw is not yet equated to the reference scale (offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; use-case and language cells are measured on its own draw.",
   "display": "Umpire 35B-A3B",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-29",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.22",
    "results_sha256": "27ba1fbbf115e51c247551557fc36b5298d609a0dbec7079e5abbfb2ca70ee47",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Self-hosted on one evaluator-rented Verda NVIDIA A100-SXM4-80GB in Finland (EU), using the author's unchanged Transformers HTTP server on loopback; BF16, no vLLM engine. Measured 29 Oct 2026. Public warm-up preceded timed measurements. Model, source and input were read-only, network cut before scored input. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 58.554135725832396,
    "I_sealed": 57.821803536347446,
    "base": 58.18796963108992,
    "excess": -1.8410554529532939,
    "gap": 0.7323321894849499,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 69.4028533253554,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 66.02705253224286,
       "judge": 72.31377719182598,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 56.97584541062802,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 65.21739130434783,
       "judge": 39.99999999999999,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 49.283708441513795,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 60.23643223977733,
       "hard": 39.73248690490482,
       "judge": 63.61408356851359,
       "standard": 41.414226925100294
      }
     },
     "sealed|choice": {
      "cc": 71.02470608460767,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 88.20831968555521,
       "hard": 55.313754489752796,
       "judge": 86.54012079378775,
       "standard": 70.58168041007357
      }
     },
     "sealed|noul": {
      "cc": 43.250676420156346,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 12.195121951219523,
       "judge": 51.11111111111111,
       "standard": 72.09302325581395
      }
     },
     "sealed|score": {
      "cc": 59.19002810427833,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 88.87512053616724,
       "hard": 32.664394114996234,
       "judge": 74.88090492242422,
       "standard": 73.86243463967924
      }
     }
    }
   },
   "jevbench_score": 74.03281814539581,
   "key": "umpire-35b-a3b",
   "last_measured_on": "2026-10-29",
   "licence": "Weights CC BY-NC 4.0 (non-commercial use only); serving code Apache-2.0 (MIT SemIf notice preserved)",
   "listing": "unranked",
   "measured_in": "v1.7.22",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": {
    "base": "ornith-ai/Ornith-1.5-35B-A3B",
    "model": "ezoushen/umpire-35b-a3b-bf16",
    "revision": "c928afd208de9f85d2c194042364b1a62ca4d080",
    "served_model": "Umpire 35B-A3B"
   },
   "new_in": "v1.7.22",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale; stronger anchor runs for that draw are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.9546925566343042,
    "decisive": 309,
    "decisive_rate": 0.8262032085561497,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "provider": "self-hosted",
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/ezoushen/umpire-35b-a3b-bf16",
   "scores": {
    "A": 74.03281814539581,
    "B": 70.2091633108012,
    "C": 69.62868414097892
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.9599967839998499,
    "p50_s_raw": 0.40499839199992493,
    "p95_s_adjusted": 1.1374826734003816,
    "p95_s_raw": 0.49374133670019077
   },
   "status": {
    "answered_ok": 1495,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.75,
       "cc": 66.66666666666666,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 66.66666666666666
      },
      "calibration_gen": {
       "accuracy": 0.5714285714285714,
       "cc": 59.98991007292196,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 100,
        "score": 31.917782166817833
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 59.98991007292196
      },
      "composite_gen": {
       "accuracy": 0.631578947368421,
       "cc": 28.114597057279592,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -19.999999999999996,
        "score": 31.010457838505424
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 28.114597057279592
      },
      "fairness": {
       "accuracy": 1,
       "cc": 95.86700063890373,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 87.60100191671118
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 95.86700063890373
      },
      "jaggedness": {
       "accuracy": 0.45454545454545453,
       "cc": 5.74624005340015,
       "cc_by_type": {
        "choice": 25.92592592592593,
        "noul": -33.333333333333336,
        "score": 24.646127567607856
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 5.74624005340015
      },
      "llm_router": {
       "accuracy": 0.8666666666666667,
       "cc": 78.88723844234805,
       "cc_by_type": {
        "choice": 78.84841363102231,
        "noul": 91.30434782608697,
        "score": 66.50895386993486
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 78.88723844234805
      },
      "long_state": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 100
      },
      "multilingual": {
       "accuracy": 0.7850162866449512,
       "cc": 66.22303105675404,
       "cc_by_type": {
        "choice": 68.63548178486744,
        "noul": 63.934426229508205,
        "score": 66.09918515588649
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 66.22303105675404
      },
      "negation_pair_gen": {
       "accuracy": 0.7727272727272727,
       "cc": 58.333333333333336,
       "cc_by_type": {
        "choice": 100,
        "noul": 16.666666666666675
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 58.333333333333336
      },
      "option_order": {
       "accuracy": 0.9629629629629629,
       "cc": 95.23809523809526,
       "cc_by_type": {
        "choice": 95.23809523809526
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 95.23809523809526
      },
      "perturb_pair": {
       "accuracy": 0.8928571428571429,
       "cc": 80.79934687248692,
       "cc_by_type": {
        "choice": 90.56603773584905,
        "noul": 80,
        "score": 71.83200288161171
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 80.79934687248692
      },
      "usecase": {
       "accuracy": 0.7292307692307692,
       "cc": 52.063212298585086,
       "cc_by_type": {
        "choice": 64.98905908096278,
        "noul": 32.30769230769231,
        "score": 58.89288550710017
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 52.063212298585086
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8571428571428571,
       "cc": 87.518973367497,
       "cc_by_type": {
        "choice": 72.6027397260274,
        "noul": 100,
        "score": 89.95418037646363
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 87.518973367497
      },
      "cs": {
       "accuracy": 0.7027027027027027,
       "cc": 51.34686268827166,
       "cc_by_type": {
        "choice": 59.89304812834225,
        "noul": 33.33333333333333,
        "score": 60.81420660313939
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 51.34686268827166
      },
      "da": {
       "accuracy": 0.6666666666666666,
       "cc": 32.4476643459059,
       "cc_by_type": {
        "choice": 70.18633540372672,
        "noul": -9.090909090909093,
        "score": 36.24756672490006
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 32.4476643459059
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 76.20290861717864,
       "cc_by_type": {
        "choice": 72.21507589400566,
        "noul": 75,
        "score": 81.39364995753022
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 76.20290861717864
      },
      "el": {
       "accuracy": 0.8214285714285714,
       "cc": 69.49394546520196,
       "cc_by_type": {
        "choice": 65.38461538461539,
        "noul": 80,
        "score": 63.0972210109905
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 69.49394546520196
      },
      "en": {
       "accuracy": 0.7567567567567568,
       "cc": 56.57365985164278,
       "cc_by_type": {
        "choice": 70.77922077922076,
        "noul": 40.98360655737705,
        "score": 57.95815221833052
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 56.57365985164278
      },
      "es": {
       "accuracy": 0.8648648648648649,
       "cc": 73.8770006664299,
       "cc_by_type": {
        "choice": 85.56535685645551,
        "noul": 77.77777777777777,
        "score": 58.2878673650564
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 73.8770006664299
      },
      "fi": {
       "accuracy": 0.7884615384615384,
       "cc": 71.59471229194651,
       "cc_by_type": {
        "choice": 58.85714285714286,
        "noul": 78.94736842105263,
        "score": 76.97962559764402
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 71.59471229194651
      },
      "fr": {
       "accuracy": 0.8125,
       "cc": 75.32944378864296,
       "cc_by_type": {
        "choice": 69.38775510204081,
        "noul": 100,
        "score": 56.6005762638881
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 75.32944378864296
      },
      "hi": {
       "accuracy": 0.6486486486486487,
       "cc": 43.25327726550004,
       "cc_by_type": {
        "choice": 66.94214876033057,
        "noul": -11.111111111111116,
        "score": 73.92879414728067
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 43.25327726550004
      },
      "id": {
       "accuracy": 0.8292682926829268,
       "cc": 74.57733724815385,
       "cc_by_type": {
        "choice": 72.35023041474655,
        "noul": 83.33333333333333,
        "score": 68.04844799638165
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 74.57733724815385
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 75.61902219920749,
       "cc_by_type": {
        "choice": 84.02839396628217,
        "noul": 83.33333333333333,
        "score": 59.49533929800695
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 75.61902219920749
      },
      "ja": {
       "accuracy": 0.75,
       "cc": 61.99906383547002,
       "cc_by_type": {
        "choice": 61.139896373057,
        "noul": 63.63636363636365,
        "score": 61.22093149698941
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 61.99906383547002
      },
      "ko": {
       "accuracy": 0.7209302325581395,
       "cc": 53.2339578762076,
       "cc_by_type": {
        "choice": 55.88235294117647,
        "noul": 39.99999999999999,
        "score": 63.81952068744632
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 53.2339578762076
      },
      "mixed": {
       "accuracy": 0.90625,
       "cc": 78.06626284217755,
       "cc_by_type": {
        "choice": 83.79837983798379,
        "noul": 100,
        "score": 50.40040868854889
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 78.06626284217755
      },
      "nl": {
       "accuracy": 0.8333333333333334,
       "cc": 46.810703043915055,
       "cc_by_type": {
        "choice": 85.84905660377359,
        "noul": 60.00000000000001,
        "score": -5.416947472028433
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 46.810703043915055
      },
      "no": {
       "accuracy": 0.7647058823529411,
       "cc": 61.08866430132247,
       "cc_by_type": {
        "choice": 58.691910499139425,
        "noul": 73.33333333333334,
        "score": 51.240749071494626
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 61.08866430132247
      },
      "pl": {
       "accuracy": 0.7192982456140351,
       "cc": 55.17392695301711,
       "cc_by_type": {
        "choice": 61.681745609366686,
        "noul": 52.941176470588225,
        "score": 50.898858779096415
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 55.17392695301711
      },
      "pt": {
       "accuracy": 0.7446808510638298,
       "cc": 61.95323729652559,
       "cc_by_type": {
        "choice": 64.05990016638934,
        "noul": 50,
        "score": 71.79981172318742
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 61.95323729652559
      },
      "sv": {
       "accuracy": 0.782608695652174,
       "cc": 69.98255319828816,
       "cc_by_type": {
        "choice": 53.07262569832402,
        "noul": 83.33333333333333,
        "score": 73.54170056320714
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 69.98255319828816
      },
      "tr": {
       "accuracy": 0.7391304347826086,
       "cc": 60.626704394727575,
       "cc_by_type": {
        "choice": 52.109464082098064,
        "noul": 39.99999999999999,
        "score": 89.77064910208466
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 60.626704394727575
      },
      "uk": {
       "accuracy": 0.7678571428571429,
       "cc": 62.362108155385634,
       "cc_by_type": {
        "choice": 72.6027397260274,
        "noul": 71.42857142857142,
        "score": 43.05501331155806
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 62.362108155385634
      },
      "zh": {
       "accuracy": 0.8780487804878049,
       "cc": 75.25147554579588,
       "cc_by_type": {
        "choice": 81.40495867768595,
        "noul": 75,
        "score": 69.34946795970168
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 75.25147554579588
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.776,
       "cc": 69.44559517434398,
       "cc_by_type": {
        "choice": 69.44559517434398
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 69.44559517434398
      },
      "noul": {
       "accuracy": 0.7866666666666666,
       "cc": 57.33333333333333,
       "cc_by_type": {
        "noul": 57.33333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 57.33333333333333
      },
      "score": {
       "accuracy": 0.76,
       "cc": 63.070542873336535,
       "cc_by_type": {
        "score": 63.070542873336535
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 63.070542873336535
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6875,
       "cc": 50.746837419440055,
       "cc_by_type": {
        "choice": 55.52260934025203,
        "noul": 14.28571428571428,
        "score": 82.43218863235384
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 50.746837419440055
      },
      "code_linting": {
       "accuracy": 0.8695652173913043,
       "cc": 78.88923372649073,
       "cc_by_type": {
        "choice": 81.62890385793018,
        "noul": 100,
        "score": 55.038797321542
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 78.88923372649073
      },
      "customer_support": {
       "accuracy": 0.7575757575757576,
       "cc": 56.946829283806686,
       "cc_by_type": {
        "choice": 66.78966789667898,
        "noul": 33.33333333333333,
        "score": 70.71748662140776
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 56.946829283806686
      },
      "demand_forecasting": {
       "accuracy": 0.42424242424242425,
       "cc": -5.6399828345693495,
       "cc_by_type": {
        "choice": 22.668688400303264,
        "noul": -53.84615384615385,
        "score": 14.257516942142534
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7580645161290323,
       "cc": 64.13689206600212,
       "cc_by_type": {
        "choice": 70.44334975369458,
        "noul": 64.70588235294117,
        "score": 57.26144409137059
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 64.13689206600212
      },
      "feature_extraction": {
       "accuracy": 0.7361111111111112,
       "cc": 54.274935317418304,
       "cc_by_type": {
        "choice": 51.005025125628144,
        "noul": 42.85714285714286,
        "score": 68.9626379694839
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 54.274935317418304
      },
      "financial_crime": {
       "accuracy": 0.7619047619047619,
       "cc": 43.90567888312243,
       "cc_by_type": {
        "choice": 77.6714513556619,
        "noul": 9.090909090909083,
        "score": 44.954676202796286
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 43.90567888312243
      },
      "gaming": {
       "accuracy": 0.6379310344827587,
       "cc": 43.015157954059305,
       "cc_by_type": {
        "choice": 42.62948207171314,
        "noul": 33.33333333333333,
        "score": 53.08265845713143
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 43.015157954059305
      },
      "insurance_claims": {
       "accuracy": 0.6451612903225806,
       "cc": 40.892905013832795,
       "cc_by_type": {
        "choice": 42.09354120267261,
        "noul": 52.941176470588225,
        "score": 27.64399736823755
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 40.892905013832795
      },
      "knowledge_graphs": {
       "accuracy": 0.7611940298507462,
       "cc": 52.46495119696531,
       "cc_by_type": {
        "choice": 77.51124437781112,
        "noul": 23.076923076923084,
        "score": 56.80668613616173
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 52.46495119696531
      },
      "lead_generation": {
       "accuracy": 0.8933333333333333,
       "cc": 71.91771399991603,
       "cc_by_type": {
        "choice": 78.84344146685473,
        "noul": 88.57142857142857,
        "score": 48.338271961464784
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 71.91771399991603
      },
      "legal_compliance": {
       "accuracy": 0.7321428571428571,
       "cc": 65.79757550337492,
       "cc_by_type": {
        "choice": 63.89167502507522,
        "noul": 71.42857142857142,
        "score": 62.072480056478106
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 65.79757550337492
      },
      "llm_guardrails": {
       "accuracy": 0.9154929577464789,
       "cc": 83.47914325195359,
       "cc_by_type": {
        "choice": 91.85336048879836,
        "noul": 92.3076923076923,
        "score": 66.2763769593701
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 83.47914325195359
      },
      "model_routing": {
       "accuracy": 0.8986486486486487,
       "cc": 81.30872714475525,
       "cc_by_type": {
        "choice": 87.53678379781894,
        "noul": 88.23529411764706,
        "score": 68.15410351879976
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 81.30872714475525
      },
      "moderation_trust_safety": {
       "accuracy": 0.8813559322033898,
       "cc": 82.63441353891432,
       "cc_by_type": {
        "choice": 80.39215686274508,
        "noul": 88.23529411764706,
        "score": 79.27578963635078
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 82.63441353891432
      },
      "other": {
       "accuracy": 0.78125,
       "cc": 62.72843840898477,
       "cc_by_type": {
        "choice": 60.04439511653718,
        "noul": 100,
        "score": 28.140920110417134
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 62.72843840898477
      },
      "recruiting": {
       "accuracy": 0.8615384615384616,
       "cc": 77.5031058134137,
       "cc_by_type": {
        "choice": 82.69646719538572,
        "noul": 84.00000000000001,
        "score": 65.81285024485537
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 77.5031058134137
      },
      "risk_assessment": {
       "accuracy": 0.6458333333333334,
       "cc": 42.12763899358924,
       "cc_by_type": {
        "choice": 43.51464435146444,
        "noul": 38.46153846153846,
        "score": 44.4067341677648
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 42.12763899358924
      },
      "scientific_discovery": {
       "accuracy": 0.6190476190476191,
       "cc": 19.080320275480727,
       "cc_by_type": {
        "choice": 48.46625766871167,
        "noul": -40,
        "score": 48.774703157730514
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 19.080320275480727
      },
      "search_retrieval": {
       "accuracy": 0.9552238805970149,
       "cc": 89.05078162978141,
       "cc_by_type": {
        "choice": 100,
        "noul": 71.42857142857142,
        "score": 95.72377346077283
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 89.05078162978141
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      85.70031764345342,
      91.32328769148326
     ],
     "I": [
      52.4438060237237,
      62.75181515373399
     ],
     "capability": [
      69.76183821427485,
      76.26774007751902
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.73266984958518,
     "capability": 74.46031974033755,
     "intelligence": 58.18796963108992
    },
    "per_type": {
     "choice": {
      "accuracy": 0.776,
      "calibration": 90.00194745866735,
      "cc": 69.44559517434398,
      "cc_by_type": {
       "choice": 69.44559517434398
      },
      "cc_tiered": 70.21377970498153,
      "cc_tiered_open": 69.4028533253554,
      "cc_tiered_sealed": 71.02470608460767,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 69.44559517434398
     },
     "noul": {
      "accuracy": 0.7866666666666666,
      "calibration": 91.3067169057382,
      "cc": 57.33333333333333,
      "cc_by_type": {
       "noul": 57.33333333333333
      },
      "cc_tiered": 50.11326091539218,
      "cc_tiered_open": 56.97584541062802,
      "cc_tiered_sealed": 43.250676420156346,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 57.33333333333333
     },
     "score": {
      "accuracy": 0.76,
      "calibration": 90.88934518434999,
      "cc": 63.070542873336535,
      "cc_by_type": {
       "score": 63.070542873336535
      },
      "cc_tiered": 54.23686827289606,
      "cc_tiered_open": 49.283708441513795,
      "cc_tiered_sealed": 59.19002810427833,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 63.070542873336535
     }
    },
    "raw_on_A": {
     "calibration": 85.43630112139586,
     "capability": 70.66795062074783,
     "intelligence": 55.899600120099805
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.005333333333333301,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.75332269011844,
    "Emphasis on Accuracy 60:20:20": 64.61612155505733,
    "Emphasis on Cost 20:20:60": 71.90649079761477,
    "Emphasis on Speed 20:60:20": 73.39051202812519,
    "Intelligence only": 58.18796963108991
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Uniphore",
   "axes": {
    "calibration": 79.97905975708014,
    "cost": 79.08243125416328,
    "intelligence": 34.15161183674326,
    "speed": 87.30790070278863
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09746896580594484,
      "ece_score": 80.50620683881104,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 80.50620683881104,
      "score_before_time_limit": 80.50620683881104,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.2815013404825737,
      "brier": 0.17946280164043968,
      "ece": 0.12226593407275449,
      "n": 373,
      "score": 75.5468131854491,
      "score_before_time_limit": 75.5468131854491,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9489247311827957,
      "interval90_width": 0.5589157706093193,
      "n": 372,
      "nrps": 0.11761783656774456,
      "score": 83.88415924698029,
      "score_before_time_limit": 83.88415924698029,
      "time_limit_failed_share": 0,
      "top_ece": 0.10234948924632484
     }
    },
    "score": 79.97905975708014
   },
   "capability": 57.0653357969117,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 752,642 input + 0 output tokens over 1,485 decisions = 506.82962963 tokens/decision. GPU-time cost USD 0.00498026315758042/1,000; old list-price cost USD 0.0152048888888889/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00498026315758042/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.015052840000000001
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004980263157580418
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Uniphore-4B-Decider",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (H100), offline read-only container (--network none); author's inference.py DecisionModel.predict_jev_task unchanged (bf16, sdpa, one forward pass, softmax over single-token option codes, temperatures.json binary T 1.0353 / choice 1.0) behind a first-party loopback TypeSafe shim; transformers 5.17 reference PyTorch kernels (no flash-linear-attention / causal-conv1d in the image) paid evaluation (public). Merged text-only Qwen3.5-4B LoRA-SFT checkpoint (Qwen3_5ForCausalLM). Runtime vllm/vllm-openai:v0.31.0 image (torch 2.13, transformers 5.17; vLLM unused) instead of the documented torch 2.8 / transformers 5.12 + fla kernels, so latency was measured on the reference PyTorch linear-attention path. Requests over the documented 16,384-token wrapper limit answer HTTP 422 (refused, not truncated). Choice options are presented in the harness's criteria order (labels are not sent), unlike the author's historical label-order evaluator. Output tokens are 0 (no generation). The author discloses that the public JevBench set was inspected during development.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 40.125336699801856,
    "I_sealed": 28.17788697368465,
    "base": 34.15161183674326,
    "excess": 2.107922955761495,
    "gap": 11.947449726117206,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 58.31864130561554,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 76.64864864864865,
       "hard": 54.70273670965713,
       "judge": 64.40342781806197,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 22.505175983436857,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 0,
       "standard": 33.33333333333333
      }
     },
     "open|score": {
      "cc": 39.55219281035318,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.70208847387271,
       "hard": 38.36737803800065,
       "judge": 36.095989014069396,
       "standard": 39.531180217724135
      }
     },
     "sealed|choice": {
      "cc": 50.50756397649687,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 67.5984752223634,
       "hard": 41.713889210130674,
       "judge": 53.28109696376104,
       "standard": 55.38915840539971
      }
     },
     "sealed|noul": {
      "cc": 11.54997858549068,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 27.586206896551737,
       "hard": 9.090909090909083,
       "judge": -7.692307692307687,
       "standard": 37.31343283582089
      }
     },
     "sealed|score": {
      "cc": 22.4761183590664,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 33.82756717731815,
       "hard": 17.222503109566212,
       "judge": 17.372987543179363,
       "standard": 34.962320672771455
      }
     }
    }
   },
   "jevbench_score": 28.32484800212957,
   "key": "uniphore-4b-decider",
   "last_measured_on": "2026-10-12",
   "licence": "not stated in the model card (no licence field; base Qwen/Qwen3.5-4B Apache-2.0)",
   "listing": "ranked",
   "measured_in": "v1.6.11",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "Uniphore/uniphore-4b-decider@2e22ed63ae0f9de590bbd368f3bfa7c678f78e9d",
   "noul_decisive": {
    "acc_among_decisive": 0.8246268656716418,
    "decisive": 268,
    "decisive_rate": 0.7184986595174263,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.97905975708014,
     "cost": 64.54050247464266,
     "intelligence": 34.15161183674326,
     "speed": 87.70990379280212
    },
    "board_release": "v1.6.43",
    "capability": 57.0653357969117,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 71,
    "capability_rank": 97,
    "composite_ci95": {
     "A": [
      16.39619384096642,
      37.316496815173174
     ],
     "B": [
      13.883211553134624,
      33.43171289678618
     ],
     "C": [
      11.386245722893339,
      25.91423389942581
     ]
    },
    "jevbench_score": 27.171470959871687,
    "measured_in": "v1.6.11",
    "method": "v1.6",
    "open_board_rank": 74,
    "rank": 107,
    "ranks": {
     "A": 107,
     "B": 110,
     "C": 119,
     "capability": 151
    },
    "scores": {
     "A": 27.171470959871687,
     "B": 23.81217493415051,
     "C": 18.86907705546645
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3529104620451108,
     "p50_s_raw": 0.10145523102255538,
     "p95_s_adjusted": 0.4801147931138985,
     "p95_s_raw": 0.16505739655694926
    },
    "views": {
     "Balanced 33:33:33": 24.914288994020374,
     "Emphasis on Accuracy 60:20:20": 20.330194245515276,
     "Emphasis on Cost 20:20:60": 26.761533337866197,
     "Emphasis on Speed 20:60:20": 29.535256858217686,
     "Intelligence only": 15.932855168800014
    }
   },
   "rank": 168,
   "ranked": true,
   "ranks": {
    "A": 168,
    "B": 169,
    "C": 171,
    "capability": 183
   },
   "repo": "https://huggingface.co/Uniphore/uniphore-4b-decider",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 28.32484800212957,
    "B": 24.51194836486697,
    "C": 19.670033334812207
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.36780445810873064,
    "p50_s_raw": 0.10890222905436531,
    "p95_s_adjusted": 0.5053507804637775,
    "p95_s_raw": 0.17767539023188872
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.6818181818181818,
       "cc": 46.27803208099471,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100,
        "score": -19.499237090349197
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 46.27803208099471
      },
      "calibration_gen": {
       "accuracy": 0.43564356435643564,
       "cc": 14.180095745864966,
       "cc_by_type": {
        "choice": 20.466595970307537,
        "noul": 19.999999999999996,
        "score": 2.0736912672873586
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 14.180095745864966
      },
      "composite_gen": {
       "accuracy": 0.5247524752475248,
       "cc": 14.724448695924453,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -33.333333333333336,
        "score": 10.027004624358726
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 14.724448695924453
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 84.90026374908027,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 54.7007912472408
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 84.90026374908027
      },
      "jaggedness": {
       "accuracy": 0.37222222222222223,
       "cc": 1.9908303184659708,
       "cc_by_type": {
        "choice": 19.15303176130896,
        "noul": -18.918918918918916,
        "score": 5.738378113007869
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 1.9908303184659708
      },
      "llm_router": {
       "accuracy": 0.7603833865814696,
       "cc": 63.6252672621707,
       "cc_by_type": {
        "choice": 63.546711279387154,
        "noul": 69.0721649484536,
        "score": 58.25692555867135
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 63.6252672621707
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.37037037037037035,
       "cc": -12.323508173824118,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -50,
        "score": -0.3725863771424587
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6036363636363636,
       "cc": 30.98148799735982,
       "cc_by_type": {
        "choice": 56.9611998695794,
        "noul": -5.555555555555558,
        "score": 41.53881967805562
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 30.98148799735982
      },
      "negation_pair_gen": {
       "accuracy": 0.47674418604651164,
       "cc": 2.037509626501965,
       "cc_by_type": {
        "choice": 50,
        "noul": -26.66666666666667,
        "score": -17.220804453827434
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 2.037509626501965
      },
      "option_order": {
       "accuracy": 0.9166666666666666,
       "cc": 89.34753661784286,
       "cc_by_type": {
        "choice": 89.34753661784286
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 89.34753661784286
      },
      "perturb_pair": {
       "accuracy": 0.765625,
       "cc": 74.81837004423572,
       "cc_by_type": {
        "choice": 63.96396396396397,
        "noul": 100,
        "score": 60.49114616874321
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 74.81837004423572
      },
      "usecase": {
       "accuracy": 0.5227272727272727,
       "cc": 25.901229344281592,
       "cc_by_type": {
        "choice": 49.75490196078433,
        "noul": 0,
        "score": 27.94878607206045
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 25.901229344281592
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 17.704048175114554,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50,
        "score": -20.69737928418014
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 17.704048175114554
      },
      "cs": {
       "accuracy": 0.2857142857142857,
       "cc": 47.368421052631575,
       "cc_by_type": {
        "choice": -5.263157894736845,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 47.368421052631575
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 17.895602852705036,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 7.757469080388535
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 17.895602852705036
      },
      "de": {
       "accuracy": 0.78125,
       "cc": 69.8201611580944,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 58.35428878401772
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 69.8201611580944
      },
      "el": {
       "accuracy": 0.3333333333333333,
       "cc": 15.14372656235736,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -66.66666666666667,
        "score": 95.896729035303
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 15.14372656235736
      },
      "en": {
       "accuracy": 0.578627280625543,
       "cc": 31.996951914881567,
       "cc_by_type": {
        "choice": 51.2332043787603,
        "noul": 20.863309352517987,
        "score": 23.89434201336642
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 31.996951914881567
      },
      "es": {
       "accuracy": 0.8709677419354839,
       "cc": 70.78955082120461,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 22.156580685473525
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 70.78955082120461
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 41.781260838599806,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 57.58421967426936
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 41.781260838599806
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 40.776530517809675,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 64.6825327298996
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 40.776530517809675
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": -0.0880455013773845,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -50,
        "score": 16.77496964111924
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 73.53419736027718,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 20.602592080831517
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 73.53419736027718
      },
      "ja": {
       "accuracy": 0.56,
       "cc": 28.657352573898077,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 19.999999999999996,
        "score": 8.863241200645778
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 28.657352573898077
      },
      "ko": {
       "accuracy": 0.4,
       "cc": 4.525100120476314,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 13.575300361428944
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 4.525100120476314
      },
      "mixed": {
       "accuracy": 0.8571428571428571,
       "cc": 72.43961465493258,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 83.98551063146442
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 72.43961465493258
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 24.53556347288955,
       "cc_by_type": {
        "choice": 22.077922077922082,
        "noul": -33.333333333333336,
        "score": 84.8621016740799
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 24.53556347288955
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6153846153846154,
       "cc": 25.142832279070205,
       "cc_by_type": {
        "choice": 69.53025814642403,
        "noul": -40,
        "score": 45.89823869078658
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 25.142832279070205
      },
      "pt": {
       "accuracy": 0.6785714285714286,
       "cc": 51.72941097740019,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 60.00000000000001,
        "score": 48.22300016849992
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 51.72941097740019
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 10.795858765536167,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": -19.999999999999996,
        "score": 54.515235871076584
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 10.795858765536167
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 18.82565444321844,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 50,
        "score": -50.04477580077946
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 18.82565444321844
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 73.86846162619129,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 53.86344939470291
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 73.86846162619129
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -6.045504219096912,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": -33.333333333333336,
        "score": -33.95572169683876
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6333333333333333,
       "cc": 52.63973669607149,
       "cc_by_type": {
        "choice": 52.63973669607149
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 52.63973669607149
      },
      "noul": {
       "accuracy": 0.5893333333333334,
       "cc": 17.866666666666674,
       "cc_by_type": {
        "noul": 17.866666666666674
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 17.866666666666674
      },
      "score": {
       "accuracy": 0.5013333333333333,
       "cc": 27.205330671354698,
       "cc_by_type": {
        "score": 27.205330671354698
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 27.205330671354698
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.4090909090909091,
       "cc": -9.794702331809445,
       "cc_by_type": {
        "choice": 36,
        "noul": -50,
        "score": -15.384106995428336
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5,
       "cc": 14.86522245682505,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": 8.018717750047266
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 14.86522245682505
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 15.191320236519909,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 0,
        "score": -21.905714087188244
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 15.191320236519909
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -14.546261990141502,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -100,
        "score": 38.232559058815255
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.4117647058823529,
       "cc": 2.453084074518673,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -33.333333333333336,
        "score": 27.649107296019793
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 2.453084074518673
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -35.326337874466766,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": -66.66666666666667,
        "score": -76.41926519572735
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 57.88596023856343,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 71.42857142857142,
        "score": 31.64107399300122
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 57.88596023856343
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -1.762092816941351,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -50,
        "score": 17.884453256493018
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 16.529699165925933,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -19.999999999999996,
        "score": 32.0890974977778
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 16.529699165925933
      },
      "knowledge_graphs": {
       "accuracy": 0.3181818181818182,
       "cc": 7.696866416285641,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": 0,
        "score": 25.874325158921163
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 7.696866416285641
      },
      "lead_generation": {
       "accuracy": 0.56,
       "cc": 29.10220149075188,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": -11.111111111111116,
        "score": 10.045622560110957
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 29.10220149075188
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -26.283415618075534,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -33.333333333333336,
        "score": -48.742719972506166
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 24.557631257691053,
       "cc_by_type": {
        "choice": 53.93474088291747,
        "noul": -19.999999999999996,
        "score": 39.738152890155696
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 24.557631257691053
      },
      "model_routing": {
       "accuracy": 0.783410138248848,
       "cc": 65.14331381778801,
       "cc_by_type": {
        "choice": 70.7792207792208,
        "noul": 78.78787878787878,
        "score": 45.86284188626449
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 65.14331381778801
      },
      "moderation_trust_safety": {
       "accuracy": 0.5555555555555556,
       "cc": -0.15637279529120462,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": -100,
        "score": 57.112455510479556
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 54.82621163125395,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 47.85839348243408
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 54.82621163125395
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -10.939838270553587,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": -100,
        "score": 11.515460557797375
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 3.5292829705879094,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 1.892196737850682
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 3.5292829705879094
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 73.5250465829234,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 50,
        "score": 81.25129633240009
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 73.5250465829234
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      74.74001355777733,
      82.37757108771744
     ],
     "I": [
      27.86490456594573,
      38.9213316269163
     ],
     "capability": [
      52.347472852953594,
      59.90975069911703
     ],
     "composite": null,
     "seed": 79
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.97905975708014,
     "capability": 57.0653357969117,
     "intelligence": 34.15161183674326
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6333333333333333,
      "calibration": 80.50620683881104,
      "cc": 52.63973669607149,
      "cc_by_type": {
       "choice": 52.63973669607149
      },
      "cc_tiered": 54.413102641056206,
      "cc_tiered_open": 58.31864130561554,
      "cc_tiered_sealed": 50.50756397649687,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 52.63973669607149
     },
     "noul": {
      "accuracy": 0.5893333333333334,
      "calibration": 75.5468131854491,
      "cc": 17.866666666666674,
      "cc_by_type": {
       "noul": 17.866666666666674
      },
      "cc_tiered": 17.027577284463767,
      "cc_tiered_open": 22.505175983436857,
      "cc_tiered_sealed": 11.54997858549068,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 17.866666666666674
     },
     "score": {
      "accuracy": 0.5013333333333333,
      "calibration": 83.88415924698029,
      "cc": 27.205330671354698,
      "cc_by_type": {
       "score": 27.205330671354698
      },
      "cc_tiered": 31.01415558470979,
      "cc_tiered_open": 39.55219281035318,
      "cc_tiered_sealed": 22.4761183590664,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 27.205330671354698
     }
    },
    "raw_on_A": {
     "calibration": 81.83637520750273,
     "capability": 58.75237966723043,
     "intelligence": 35.66838412695813
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 26.219567929864514,
    "Emphasis on Accuracy 60:20:20": 20.838096480742227,
    "Emphasis on Cost 20:20:60": 29.651262277897626,
    "Emphasis on Speed 20:60:20": 30.577340386529404,
    "Intelligence only": 15.932855168800014
   }
  },
  {
   "a4": {
    "measured_on": "2026-10-06",
    "n_items": 600,
    "n_ok": 591,
    "round": "score-a4-2",
    "subset": "A4"
   },
   "api_flag": true,
   "author": "Vansa",
   "axes": {
    "calibration": 86.52,
    "cost": 61.92,
    "intelligence": 40.85,
    "speed": 95.05
   },
   "calibration": {
    "score": 86.52
   },
   "capability": 63.68,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     25.3,
     68.97
    ],
    "B": [
     21.98,
     64.42
    ],
    "C": [
     17.57,
     49.9
    ]
   },
   "cost": {
    "basis": "measured: token usage of the 6 Oct 2026 A4 u P run x tariff USD 0.034/M input, 0/M output (spec vansa-3.4.json), per 1,000 decisions on the v1.6.1 common cost basis (591 of 600 items; the 9 A4 u P items outside the Jev reference's accepted input range are excluded, cost-basis-a4.json)",
    "kind": "tariff",
    "usd_per_1000": 0.018590751269035533
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 600,
    "treatment": "share"
   },
   "display": "Vansa-3.4 (Vansa, hosted System One API)",
   "endpoint_kind": "api",
   "intelligence": {
    "I_open": 43.36,
    "I_sealed": 33.19,
    "base": 40.85,
    "per_type_split": {
     "open|choice": {
      "cc": 62.8658,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.4324,
       "hard": 56.5901,
       "judge": 64.4034,
       "standard": 62.3273
      }
     },
     "open|noul": {
      "cc": 32.7674,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.1429,
       "hard": 56.5217,
       "judge": 0,
       "standard": 22.2222
      }
     },
     "open|score": {
      "cc": 34.447,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 54.8152,
       "hard": 36.7794,
       "judge": 37.1156,
       "standard": 15.5952
      }
     },
     "sealed|choice": {
      "cc": 60.2137,
      "n": {
       "easy": 23,
       "hard": 66,
       "judge": 28,
       "standard": 33
      },
      "tiers": {
       "easy": 66.6358,
       "hard": 45.7072,
       "judge": 63.8191,
       "standard": 80.6076
      }
     },
     "sealed|noul": {
      "cc": 21.254,
      "n": {
       "easy": 17,
       "hard": 22,
       "judge": 20,
       "standard": 16
      },
      "tiers": {
       "easy": 41.1765,
       "hard": 9.0909,
       "judge": 20,
       "standard": 37.5
      }
     },
     "sealed|score": {
      "cc": 18.1016,
      "n": {
       "easy": 11,
       "hard": 32,
       "judge": 11,
       "standard": 21
      },
      "tiers": {
       "easy": 53.2589,
       "hard": -0.6743,
       "judge": 4.7068,
       "standard": 58.1668
      }
     }
    }
   },
   "jevbench_score": 42.57,
   "key": "vansa-3.4",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary hosted API",
   "listing": "ranked",
   "method": "v1.7",
   "open": "no",
   "previous": {
    "api_board_rank": 27,
    "axes": {
     "calibration": 86.52,
     "cost": 61.92,
     "intelligence": 40.85,
     "speed": 95.49
    },
    "board_release": "v1.6.43",
    "capability": 63.68,
    "capability_api_board_rank": 23,
    "capability_open_board_rank": null,
    "capability_rank": 66,
    "composite_ci95": {
     "A": [
      25.32,
      69.03
     ],
     "B": [
      21.99,
      64.46
     ],
     "C": [
      17.58,
      49.94
     ]
    },
    "jevbench_score": 42.6,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 75,
    "ranks": {
     "A": 75,
     "B": 79,
     "C": 79,
     "capability": 108
    },
    "scores": {
     "A": 42.6,
     "B": 38.29,
     "C": 29.58
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 591,
     "p50_s_adjusted": 0.15,
     "p50_s_raw": 0.15,
     "p95_s_adjusted": 0.189,
     "p95_s_raw": 0.189
    },
    "views": null
   },
   "rank": 142,
   "ranked": true,
   "ranks": {
    "A": 142,
    "B": 144,
    "C": 147,
    "capability": 139
   },
   "repo": "https://vansa.org",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 42.57,
    "B": 38.27,
    "C": 29.56
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 600,
    "p50_s_adjusted": 0.158,
    "p50_s_raw": 0.158,
    "p95_s_adjusted": 0.198,
    "p95_s_raw": 0.198
   },
   "status": {
    "answered_ok": 591,
    "rows": 600,
    "status": "complete"
   },
   "v16": {
    "api_subset_tag": "A4",
    "equated": true,
    "equating_offset": {
     "C": 3.349920853296446,
     "I": 2.5713984460596446
    },
    "full_set_api": false,
    "lane": "api",
    "measured_on": "2026-10-06",
    "n_items": 600,
    "round": "score-a4-2",
    "run_sha256": "84165e0500a8ebbc881b3cd1115354b060b09c1d964d82e41df5245a2cb22c6e"
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Nandakishor M",
   "axes": {
    "calibration": 66.9335611044975,
    "cost": 85.77193998006749,
    "intelligence": 5.342898619124496,
    "speed": 73.05648557202497
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.16337254738540735,
      "ece_score": 67.32549052291853,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 66.42781731594629,
      "score_before_time_limit": 67.32549052291853,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.8053333333333333,
      "brier": 0.28665151355378216,
      "ece": 0.18020482243198754,
      "n": 375,
      "score": 63.61792065752995,
      "score_before_time_limit": 63.959035513602494,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.8106666666666666,
      "interval90_width": 0.5722222222222216,
      "n": 375,
      "nrps": 0.18890773137678013,
      "score": 70.75494534001628,
      "score_before_time_limit": 71.32554973791963,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.1922906369324137
     }
    },
    "score": 66.9335611044975
   },
   "capability": 36.138229861811,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.8B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 732,071 input + 0 output tokens over 1,485 decisions = 492.977104377 tokens/decision. GPU-time cost USD 0.00298035959729571/1,000; old list-price cost USD 0.00492977104377104/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00298035959729571/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.012253273333333332
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0029803595972957087
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Vega-U (Qwen3.5-0.8B reader)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-18",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "our own CPU server (EPYC 7502P, 32 of 64 threads, 96 GB), disposable docker container with --network none (cut proven from inside before the sealed mount), --cap-drop=ALL, no-new-privileges, unprivileged user, one read-only sealed input mount; reviewed vegaml==0.12.2, torch 2.14.1+cpu, transformers 5.19.0; fp32 throughout (the author's own backbone_dtype() choice on CPU); context 73,728, and a longer state is cut on the right by the author's own build_prompt - 15 of the 1,500 items sit at that cap and are answered on a truncated state. Reference PyTorch paths only.",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 6.448296966828988,
    "I_sealed": 4.237500271420004,
    "base": 5.342898619124496,
    "excess": -7.628730074946727,
    "gap": 2.210796695408984,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 19.344890900486963,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 10.486486486486495,
       "hard": 9.405473419314257,
       "judge": 16.94133157547792,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -57.14285714285714,
       "hard": -82.6086956521739,
       "judge": -70,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 23.713101067178897,
       "hard": -16.932168774632817,
       "judge": -31.207540516067354,
       "standard": -9.704346143973197
      }
     },
     "sealed|choice": {
      "cc": 12.71250081426001,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 18.04320203303686,
       "hard": 11.009598704753081,
       "judge": 14.495592556317357,
       "standard": 10.778316810799424
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -72.41379310344827,
       "hard": -83.63636363636364,
       "judge": -81.53846153846153,
       "standard": -82.08955223880598
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 1.711176163732453,
       "hard": -8.801799755235983,
       "judge": -15.186794142496286,
       "standard": -9.891030118368649
      }
     }
    }
   },
   "jevbench_score": 0.2008099765046913,
   "key": "vega-u-qwen3.5-0.8b",
   "last_measured_on": "2026-10-18",
   "licence": "vega-U checkpoint apache-2.0 (card and metadata agree); vegaml package apache-2.0; frozen reader Qwen/Qwen3.5-0.8B apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.29",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nandakishorm/vega-U@b3e7383e463ef3b78324408cd82f4ab0d8f2e75a",
   "noul_decisive": {
    "acc_among_decisive": 0.5342465753424658,
    "decisive": 73,
    "decisive_rate": 0.19571045576407506,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 67.53669192481355,
     "cost": 79.21519751338194,
     "intelligence": 5.503418464051958,
     "speed": 73.8348503635803
    },
    "board_release": "v1.6.43",
    "capability": 36.52005519443276,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.059047982307085994,
      0.5209750966462322
     ],
     "B": [
      0.039373315414271894,
      0.3690362057340607
     ],
     "C": [
      0.04100554326880973,
      0.3617882615598834
     ]
    },
    "jevbench_score": 0.21762292373052544,
    "measured_in": "v1.6.29",
    "method": "v1.6",
    "open_board_rank": 169,
    "rank": 213,
    "ranks": {
     "A": 213,
     "B": 213,
     "C": 213,
     "capability": 213
    },
    "scores": {
     "A": 0.21762292373052544,
     "B": 0.14979599675011504,
     "C": 0.15112703036842048
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.5407199749955907,
     "p50_s_raw": 0.6953599874977954,
     "p95_s_adjusted": 2.684054008999373,
     "p95_s_raw": 1.2670270044996865
    },
    "views": {
     "Balanced 33:33:33": 0.1748431414819662,
     "Emphasis on Accuracy 60:20:20": 0.10603360540342595,
     "Emphasis on Cost 20:20:60": 0.25984513652167707,
     "Emphasis on Speed 20:60:20": 0.2578104829204347,
     "Intelligence only": 0.06667416738798192
    }
   },
   "rank": 255,
   "ranked": true,
   "ranks": {
    "A": 255,
    "B": 255,
    "C": 255,
    "capability": 255
   },
   "repo": "https://huggingface.co/nandakishorm/vega-U",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.2008099765046913,
    "B": 0.13770132634063093,
    "C": 0.1394513725727023
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.6106067149725276,
    "p50_s_raw": 0.7303033574862638,
    "p95_s_adjusted": 3.071580751580768,
    "p95_s_raw": 1.460790375790384
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.3181818181818182,
       "cc": -1.8642060422197488,
       "cc_by_type": {
        "choice": 8.333333333333334,
        "noul": 0,
        "score": -13.92595145999258
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.2079207920792079,
       "cc": -19.789838255276152,
       "cc_by_type": {
        "choice": -8.165429480381752,
        "noul": -43.99999999999999,
        "score": -7.204085285446715
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.1485148514851485,
       "cc": -44.963648001333546,
       "cc_by_type": {
        "choice": -7.317073170731707,
        "noul": -100,
        "score": -27.57387083326892
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.3125,
       "cc": -18.313256734755296,
       "cc_by_type": {
        "choice": 14.285714285714292,
        "noul": -60,
        "score": -9.225484489980174
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.2111111111111111,
       "cc": -34.81620625476442,
       "cc_by_type": {
        "choice": -3.9461020211741884,
        "noul": -89.1891891891892,
        "score": -11.313327553929863
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.22364217252396165,
       "cc": -24.720816695072543,
       "cc_by_type": {
        "choice": 17.583868979484027,
        "noul": -85.56701030927836,
        "score": -6.179308755423296
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -19.02278279739924,
       "cc_by_type": {
        "choice": 38.14432989690721,
        "noul": -75,
        "score": -20.212678289104936
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.3236363636363636,
       "cc": -22.568131932211916,
       "cc_by_type": {
        "choice": 23.703945223345297,
        "noul": -77.77777777777779,
        "score": -13.630563242203264
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.23255813953488372,
       "cc": -32.29973265796035,
       "cc_by_type": {
        "choice": 3.333333333333336,
        "noul": -80,
        "score": -20.232531307214387
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.3333333333333333,
       "cc": 14.780292942743008,
       "cc_by_type": {
        "choice": 14.780292942743008
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 14.780292942743008
      },
      "perturb_pair": {
       "accuracy": 0.40625,
       "cc": -12.512300010905099,
       "cc_by_type": {
        "choice": 27.92792792792793,
        "noul": -60,
        "score": -5.464827960643226
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.2909090909090909,
       "cc": -18.881107912882545,
       "cc_by_type": {
        "choice": 22.794117647058847,
        "noul": -83.33333333333334,
        "score": 3.8958919476268616
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.25,
       "cc": -36.55525876106678,
       "cc_by_type": {
        "choice": 4.76190476190476,
        "noul": -100,
        "score": -14.4276810451051
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": -21.05263157894737,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.15384615384615385,
       "cc": -65.52162213002279,
       "cc_by_type": {
        "choice": -3.6866359447004595,
        "noul": -100,
        "score": -92.87823044536792
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.40625,
       "cc": -18.671575997460423,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": -100,
        "score": -24.156320912735254
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "el": {
       "accuracy": 0.26666666666666666,
       "cc": -32.16558414122873,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -100,
        "score": -29.45764627843479
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.24413553431798435,
       "cc": -26.01741064970663,
       "cc_by_type": {
        "choice": 9.49585677499659,
        "noul": -79.13669064748201,
        "score": -8.41139807663447
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.3225806451612903,
       "cc": -23.899883923976507,
       "cc_by_type": {
        "choice": 11.908646003262646,
        "noul": -75,
        "score": -8.608297775192163
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.25,
       "cc": -40.6825249989779,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": -56.47380450513042
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5,
       "cc": -11.687923076823088,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": -100,
        "score": 28.465642534236622
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.47619047619047616,
       "cc": -3.1055582694130988,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -75,
        "score": 15.96265480069925
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0,
       "cc": -66.66666666666667,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.4,
       "cc": 5.813112671547951,
       "cc_by_type": {
        "choice": 29.851909586905684,
        "noul": -33.333333333333336,
        "score": 20.920761761071503
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 5.813112671547951
      },
      "ja": {
       "accuracy": 0.48,
       "cc": 7.3870916359845324,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": -19.999999999999996,
        "score": -0.6504804534443354
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 7.3870916359845324
      },
      "ko": {
       "accuracy": 0.2,
       "cc": -51.878016279386806,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -100,
        "score": -22.300715504827085
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.35714285714285715,
       "cc": 5.358585021957109,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -33.333333333333336,
        "score": 16.075755065871324
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 5.358585021957109
      },
      "nl": {
       "accuracy": 0.3,
       "cc": -19.24038798812712,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -100,
        "score": -5.773112016329418
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.23076923076923078,
       "cc": -30.678147730963047,
       "cc_by_type": {
        "choice": 16.208209902666106,
        "noul": -80,
        "score": -28.242653095555248
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.35714285714285715,
       "cc": -8.753322219103685,
       "cc_by_type": {
        "choice": 25.75132586918091,
        "noul": -60,
        "score": 7.988707473508039
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0,
       "cc": -39.34001974615732,
       "cc_by_type": {
        "choice": -27.659574468085104,
        "noul": -100,
        "score": 9.63951522961315
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.3076923076923077,
       "cc": -52.30606124559736,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100,
        "score": -91.70079243244427
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.14285714285714285,
       "cc": -68.27594716841715,
       "cc_by_type": {
        "choice": 3.225806451612906,
        "noul": -100,
        "score": -108.05364795686434
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.1,
       "cc": -38.962930251919914,
       "cc_by_type": {
        "choice": -27.11864406779661,
        "noul": -100,
        "score": 10.229853312036862
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.328,
       "cc": 13.201553799345579,
       "cc_by_type": {
        "choice": 13.201553799345579
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 13.201553799345579
      },
      "noul": {
       "accuracy": 0.104,
       "cc": -79.2,
       "cc_by_type": {
        "noul": -79.2
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.2853333333333333,
       "cc": -10.082003664821503,
       "cc_by_type": {
        "score": -10.082003664821503
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.22727272727272727,
       "cc": -34.926263523751715,
       "cc_by_type": {
        "choice": 20,
        "noul": -100,
        "score": -24.778790571255136
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.20833333333333334,
       "cc": -24.119846467833653,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -100,
        "score": 2.1746220871822763
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 20.313465191128632,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 0,
        "score": 25.98104597988996
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 20.313465191128632
      },
      "demand_forecasting": {
       "accuracy": 0.23529411764705882,
       "cc": -32.196963481052016,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -100,
        "score": -3.0236389811677533
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.23529411764705882,
       "cc": -45.51840486299776,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": -49.59869284986284
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -46.93173196122063,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -66.66666666666667,
        "score": -98.65683110378765
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.3333333333333333,
       "cc": -23.154503740343333,
       "cc_by_type": {
        "choice": 31.372549019607845,
        "noul": -100,
        "score": -0.8360602406378481
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.13636363636363635,
       "cc": -46.359310004542714,
       "cc_by_type": {
        "choice": -4.529616724738673,
        "noul": -100,
        "score": -34.54831328888945
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": -17.751149481419645,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -100,
        "score": 30.079884889074393
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.22727272727272727,
       "cc": -27.831068186330523,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -100,
        "score": 19.290521351072666
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.28,
       "cc": -11.984104879552689,
       "cc_by_type": {
        "choice": 30.232558139534877,
        "noul": -100,
        "score": 33.81512722180705
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": -35.612368484434,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100,
        "score": -74.57904093717296
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 5.575609350286421,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -60,
        "score": 11.275772388671168
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 5.575609350286421
      },
      "model_routing": {
       "accuracy": 0.2534562211981567,
       "cc": -21.434671725606155,
       "cc_by_type": {
        "choice": 25.194805194805216,
        "noul": -78.78787878787878,
        "score": -10.710941583744905
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0
      },
      "moderation_trust_safety": {
       "accuracy": 0.2777777777777778,
       "cc": -35.69443881183727,
       "cc_by_type": {
        "choice": 19.385796545105567,
        "noul": -100,
        "score": -26.469112980617382
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.2222222222222222,
       "cc": -30.652547782563005,
       "cc_by_type": {
        "choice": -0.2785515320334252,
        "noul": -100,
        "score": 8.320908184344411
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 11.928697621534182,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": 0,
        "score": -5.10060664278662
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 11.928697621534182
      },
      "scientific_discovery": {
       "accuracy": 0.21739130434782608,
       "cc": -26.799833636121605,
       "cc_by_type": {
        "choice": -1.4492753623188352,
        "noul": -66.66666666666667,
        "score": -12.283558879379308
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.20833333333333334,
       "cc": -28.36783840626855,
       "cc_by_type": {
        "choice": 25.26690391459075,
        "noul": -100,
        "score": -10.370419133396403
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      60.99693003197813,
      70.28216529528869
     ],
     "I": [
      3.33432352068163,
      7.3529305629723165
     ],
     "capability": [
      32.78281449779358,
      38.263101265133066
     ],
     "composite": null,
     "seed": 80
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 66.9335611044975,
     "capability": 36.138229861811,
     "intelligence": 5.342898619124496
    },
    "per_type": {
     "choice": {
      "accuracy": 0.328,
      "calibration": 66.42781731594629,
      "cc": 13.201553799345579,
      "cc_by_type": {
       "choice": 13.201553799345579
      },
      "cc_tiered": 16.028695857373485,
      "cc_tiered_open": 19.344890900486963,
      "cc_tiered_sealed": 12.71250081426001,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 13.201553799345579
     },
     "noul": {
      "accuracy": 0.104,
      "calibration": 63.61792065752995,
      "cc": -79.2,
      "cc_by_type": {
       "noul": -79.2
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2853333333333333,
      "calibration": 70.75494534001628,
      "cc": -10.082003664821503,
      "cc_by_type": {
       "score": -10.082003664821503
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 66.70020516657101,
     "capability": 35.870650331003844,
     "intelligence": 5.0410954954366805
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.16119565035775474,
    "Emphasis on Accuracy 60:20:20": 0.09728912103700492,
    "Emphasis on Cost 20:20:60": 0.24209563413860152,
    "Emphasis on Speed 20:60:20": 0.23799984453327677,
    "Intelligence only": 0.061008562485919474
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Nandakishor M",
   "axes": {
    "calibration": 76.81388603317517,
    "cost": 79.44348865131269,
    "intelligence": 8.45673307172142,
    "speed": 88.8199586422866
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07314029135902292,
      "ece_score": 85.37194172819542,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 84.2336491718195,
      "score_before_time_limit": 85.37194172819542,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.6693333333333333,
      "brier": 0.22771527118772983,
      "ece": 0.12364099316545366,
      "n": 375,
      "score": 74.8703517596191,
      "score_before_time_limit": 75.27180136690927,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.768,
      "interval90_width": 0.4584444444444443,
      "n": 375,
      "nrps": 0.1744417912695621,
      "score": 71.33765716808692,
      "score_before_time_limit": 71.91296085492633,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.1936494958159556
     }
    },
    "score": 76.81388603317517
   },
   "capability": 42.63530955244829,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 732,071 input + 0 output tokens over 1,485 decisions = 492.977104377 tokens/decision. GPU-time cost USD 0.00484414400210599/1,000; old list-price cost USD 0.0147893131313131/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00484414400210599/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.03675982
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004844144002105987
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Vega-U (Qwen3.5-4B reader)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Verda RTX 6000 Ada (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-01, one dedicated GPU instance, network none (proven), evaluator-controlled self-hosted deployment, measured 23 Oct 2026 06:56-07:03 UTC. The declared reviewed loopback shim `python3 vega_shim.py <pinned snapshot>/ckpt <port> vega-u-qwen3.5-4b`, with `VEGA_DEVICE=cuda`, vegaml==0.12.2, torch 2.14.1+cu126, mode=engine, adapters=auto, views=1, no max_len override (73,728); vegaml chooses the GPU dtype. The author's GPU configuration is used; the evaluator's RTX 6000 Ada is the only hardware deviation. The offline container-A2 recount matches the forecast fixed before the run: 1,500 answered, 0 refused, 1,837,991 input tokens, no output tokens. Fifteen items are cut on the right at the author's 73,728-token limit and take 13.8-14.3 s each on this card, so they count as failed under the 10 s rule. Speed is a dedicated-GPU figure measured one request at a time; the standard self-hosted x2 + 0.15 s adjustment applies and raw p50/p95 are kept. RE-MEASURE ON THE SAME ITEMS: this GPU row REPLACES the earlier loaded-CPU row in full (scores AND times); the GPU times are not combined with the CPU scores. The same 1,500 items of draw v1.6-regular-20261012 (input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b), the same pinned weights. Pod: Verda 1RTX6000ADA.10V (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-01. The scoring run had no network: container NetworkMode none, only the loopback interface, cut proven before the first sealed byte; read-only root, all capabilities dropped. The earlier CPU measurement of this row was its first use of the weights on this draw; this is the second and last.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 7.801204167922121,
    "I_sealed": 9.11226197552072,
    "base": 8.45673307172142,
    "excess": -11.15058457795431,
    "gap": -1.3110578075985995,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 23.403612503766364,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 14.378378378378383,
       "hard": 28.279333123623783,
       "judge": 9.030982201713911,
       "standard": 39.723733779824194
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -30.434782608695656,
       "judge": -50,
       "standard": -55.55555555555556
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 51.312243229154305,
       "hard": -8.801569670026566,
       "judge": 0.2849655036505916,
       "standard": -17.412006597581488
      }
     },
     "sealed|choice": {
      "cc": 27.336785926562165,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 29.47903430749683,
       "hard": 24.540302995258457,
       "judge": 26.836434867776703,
       "standard": 32.609154186880424
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": -41.379310344827594,
       "hard": -50.90909090909091,
       "judge": -69.23076923076923,
       "standard": -46.26865671641791
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 20.512927693345652,
       "hard": 5.426302008135641,
       "judge": -15.699545487050193,
       "standard": -2.3208485625247466
      }
     }
    }
   },
   "jevbench_score": 0.7376917704778548,
   "key": "vega-u-qwen3.5-4b",
   "last_measured_on": "2026-10-23",
   "licence": "vega-U checkpoint apache-2.0 (card and metadata agree); vegaml package apache-2.0; frozen reader Qwen/Qwen3.5-4B apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nandakishorm/vega-U@b3e7383e463ef3b78324408cd82f4ab0d8f2e75a",
   "noul_decisive": {
    "acc_among_decisive": 0.7580645161290323,
    "decisive": 124,
    "decisive_rate": 0.3324396782841823,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.8233873834119,
     "cost": 64.90155987179207,
     "intelligence": 8.801638441109716,
     "speed": 60.651933418279626
    },
    "board_release": "v1.6.43",
    "capability": 44.31251291226081,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      0.4134795884464807,
      1.7993673499327525
     ],
     "B": [
      0.29304131966029756,
      1.366981924419596
     ],
     "C": [
      0.28713860308783384,
      1.2495606596755224
     ]
    },
    "jevbench_score": 0.7843042690267673,
    "measured_in": "v1.6.32",
    "method": "v1.6",
    "open_board_rank": 156,
    "rank": 199,
    "ranks": {
     "A": 199,
     "B": 199,
     "C": 199,
     "capability": 189
    },
    "scores": {
     "A": 0.7843042690267673,
     "B": 0.570350285841166,
     "C": 0.5446557423796995
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 6.766180534940213,
     "p50_s_raw": 3.3080902674701065,
     "p95_s_adjusted": 12.719296073674922,
     "p95_s_raw": 6.284648036837461
    },
    "views": {
     "Balanced 33:33:33": 0.638871364050114,
     "Emphasis on Accuracy 60:20:20": 0.41567103323551446,
     "Emphasis on Cost 20:20:60": 0.8786972307100303,
     "Emphasis on Speed 20:60:20": 0.8680676350289552,
     "Intelligence only": 0.27274108540540976
    }
   },
   "rank": 243,
   "ranked": true,
   "ranks": {
    "A": 243,
    "B": 243,
    "C": 243,
    "capability": 239
   },
   "remeasure": {
    "disclosure": "Re-measure on the same items: this GPU row replaces the earlier loaded-CPU row of the same weights in full (scores and times) and is the second and last use of these weights on draw v1.6-regular-20261012. Fifteen items reach the author's 73,728-token context limit and are answered on a right-truncated state; they take 13.8-14.3 s each on this GPU and count as failed under the 10 s rule.",
    "draw": "v1.6-regular-20261012",
    "kind": "re-measure on the same items",
    "replaces": {
     "board_release": "v1.6.43",
     "endpoint_kind": "cpu",
     "in_full": "scores and times"
    }
   },
   "remeasured_in": "v1.7.0",
   "repo": "https://huggingface.co/nandakishorm/vega-U",
   "scores": {
    "A": 0.7376917704778548,
    "B": 0.5232339963239429,
    "C": 0.5122859517207324
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.33023520699989606,
    "p50_s_raw": 0.09011760349994802,
    "p95_s_adjusted": 0.39735690419992126,
    "p95_s_raw": 0.12367845209996063
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.45454545454545453,
       "cc": -25.03596198376302,
       "cc_by_type": {
        "choice": 33.333333333333336,
        "noul": -50,
        "score": -58.441219284622406
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.27722772277227725,
       "cc": -14.828368777600858,
       "cc_by_type": {
        "choice": 4.559915164369044,
        "noul": -36,
        "score": -13.045021497171616
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.31683168316831684,
       "cc": -14.0069374233143,
       "cc_by_type": {
        "choice": 2.4390243902439046,
        "noul": -40.74074074074075,
        "score": -3.719095919446058
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.7083333333333334,
       "cc": 51.26191529469682,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 20.452412550757128
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 51.26191529469682
      },
      "jaggedness": {
       "accuracy": 0.2611111111111111,
       "cc": -23.663861725959578,
       "cc_by_type": {
        "choice": 4.716073147256991,
        "noul": -62.16216216216216,
        "score": -13.545496162973558
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.38338658146964855,
       "cc": 0.4762587666438864,
       "cc_by_type": {
        "choice": 27.093422558774332,
        "noul": -42.26804123711341,
        "score": 16.603394978270735
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 0.4762587666438864
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.25925925925925924,
       "cc": -53.162603064709884,
       "cc_by_type": {
        "choice": 13.402061855670107,
        "noul": -100,
        "score": -72.88987104979974
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.43636363636363634,
       "cc": -1.4153482680333316,
       "cc_by_type": {
        "choice": 48.15780893381155,
        "noul": -63.888888888888886,
        "score": 11.48503515097734
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.26744186046511625,
       "cc": -22.098041025583615,
       "cc_by_type": {
        "choice": -6.666666666666665,
        "noul": -40,
        "score": -19.627456410084186
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.3125,
       "cc": 12.117177097203731,
       "cc_by_type": {
        "choice": 12.117177097203731
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 12.117177097203731
      },
      "perturb_pair": {
       "accuracy": 0.46875,
       "cc": -6.191039931210905,
       "cc_by_type": {
        "choice": 39.93993993993994,
        "noul": -80,
        "score": 21.486940266427347
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 0
      },
      "usecase": {
       "accuracy": 0.40454545454545454,
       "cc": -4.150913529523027,
       "cc_by_type": {
        "choice": 43.627450980392176,
        "noul": -62.5,
        "score": 6.419808431038742
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.1875,
       "cc": -55.1179221238957,
       "cc_by_type": {
        "choice": -14.285714285714286,
        "noul": -50,
        "score": -101.06805208597281
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": -31.578947368421055,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.46153846153846156,
       "cc": -16.347518774147833,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -100,
        "score": -28.305229133503417
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.46875,
       "cc": 9.53194357801521,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": -50,
        "score": 10.454237813691647
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 9.53194357801521
      },
      "el": {
       "accuracy": 0.2,
       "cc": -48.12894813495393,
       "cc_by_type": {
        "choice": 16.201117318435756,
        "noul": -100,
        "score": -60.58796172329757
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.36055603822762816,
       "cc": -7.544666244633066,
       "cc_by_type": {
        "choice": 22.0170610561257,
        "noul": -45.32374100719424,
        "score": 0.6726812171693464
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.45161290322580644,
       "cc": -7.245856233278336,
       "cc_by_type": {
        "choice": 31.484502446982063,
        "noul": -25,
        "score": -28.222071146817072
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.125,
       "cc": -31.446955625689117,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -100,
        "score": 4.019788860637563
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 7.9862271549963095,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": -100,
        "score": 45.13515205322422
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 7.9862271549963095
      },
      "hi": {
       "accuracy": 0.42857142857142855,
       "cc": -5.873615988487875,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -100,
        "score": 32.65848164347492
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.25,
       "cc": -44.44444444444444,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.6,
       "cc": -12.368119435536334,
       "cc_by_type": {
        "choice": 71.94076383476228,
        "noul": -100,
        "score": -9.045122141371277
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ja": {
       "accuracy": 0.44,
       "cc": 2.520638957463568,
       "cc_by_type": {
        "choice": -0.07942811755361648,
        "noul": 0,
        "score": 7.64134498994432
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 2.520638957463568
      },
      "ko": {
       "accuracy": 0.3,
       "cc": -38.53246573373655,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": -26.708508312320767
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.5714285714285714,
       "cc": 42.35289813052079,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 33.33333333333333,
        "score": 27.058694391562376
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 42.35289813052079
      },
      "nl": {
       "accuracy": 0.4,
       "cc": -11.660456122154708,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": -9.007342392438167
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.16666666666666666,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.4358974358974359,
       "cc": 9.515495097600082,
       "cc_by_type": {
        "choice": 54.29538721963606,
        "noul": -60,
        "score": 34.25109807316419
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 9.515495097600082
      },
      "pt": {
       "accuracy": 0.39285714285714285,
       "cc": -5.205980171443635,
       "cc_by_type": {
        "choice": 36.35827931644077,
        "noul": -60,
        "score": 8.023780169228322
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 0
      },
      "sv": {
       "accuracy": 0.2727272727272727,
       "cc": 2.6759101210372713,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -60,
        "score": 44.62347504396287
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 2.6759101210372713
      },
      "tr": {
       "accuracy": 0.23076923076923078,
       "cc": -36.08796813124616,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": -100,
        "score": -43.04651308939067
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.42857142857142855,
       "cc": -27.496009583018008,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100,
        "score": -50.229964232924985
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.4,
       "cc": 9.62711250017331,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336,
        "score": 38.48585727453122
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 9.62711250017331
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.43066666666666664,
       "cc": 26.462427524445555,
       "cc_by_type": {
        "choice": 26.462427524445555
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 26.462427524445555
      },
      "noul": {
       "accuracy": 0.25066666666666665,
       "cc": -49.86666666666667,
       "cc_by_type": {
        "noul": -49.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.368,
       "cc": 0.9290025644768996,
       "cc_by_type": {
        "score": 0.9290025644768996
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0.9290025644768996
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.2727272727272727,
       "cc": -19.783631212480014,
       "cc_by_type": {
        "choice": 20,
        "noul": -75,
        "score": -4.350893637440034
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.2916666666666667,
       "cc": -24.99740795770514,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": -100,
        "score": -37.7251431277738
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 0
      },
      "customer_support": {
       "accuracy": 0.5,
       "cc": 16.845304659485688,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 0,
        "score": 15.576564384961122
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 16.845304659485688
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -42.72991178726473,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -58.014296765302944
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": -9.175745041749694,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": -65.20839454553895
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.3684210526315789,
       "cc": -37.19985828253814,
       "cc_by_type": {
        "choice": 24.528301886792455,
        "noul": -66.66666666666667,
        "score": -69.4612100677402
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.4,
       "cc": -8.951088980880352,
       "cc_by_type": {
        "choice": 50.98039215686275,
        "noul": -71.42857142857143,
        "score": -6.4050876709323745
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.3181818181818182,
       "cc": -20.737580599805494,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": -100,
        "score": 10.9579899079006
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.26666666666666666,
       "cc": -16.26424603536173,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -100,
        "score": 34.54059522724814
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.2727272727272727,
       "cc": -15.584461807455959,
       "cc_by_type": {
        "choice": -2.7837259100642404,
        "noul": -66.66666666666667,
        "score": 22.697007154363035
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.36,
       "cc": -3.5346979404213195,
       "cc_by_type": {
        "choice": 65.11627906976744,
        "noul": -100,
        "score": 24.279627108968594
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 0
      },
      "legal_compliance": {
       "accuracy": 0.2,
       "cc": -43.23093560590408,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": -100,
        "score": -32.91861326932515
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.48148148148148145,
       "cc": 6.361974315412698,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -60,
        "score": 2.1185525047793363
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 6.361974315412698
      },
      "model_routing": {
       "accuracy": 0.33640552995391704,
       "cc": -6.464313948023082,
       "cc_by_type": {
        "choice": 25.194805194805216,
        "noul": -42.42424242424242,
        "score": -2.163504614632039
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 0
      },
      "moderation_trust_safety": {
       "accuracy": 0.4444444444444444,
       "cc": -12.23020819041083,
       "cc_by_type": {
        "choice": 30.9021113243762,
        "noul": -100,
        "score": 32.40726410439131
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.2777777777777778,
       "cc": -1.8346473851051428,
       "cc_by_type": {
        "choice": 16.434540389972145,
        "noul": -33.333333333333336,
        "score": 11.394850788045762
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 0
      },
      "risk_assessment": {
       "accuracy": 0.3333333333333333,
       "cc": -15.876737884882187,
       "cc_by_type": {
        "choice": 26.10837438423645,
        "noul": -100,
        "score": 26.261411961117
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.34782608695652173,
       "cc": -10.815448509295445,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": -26.649244078610977
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.625,
       "cc": 27.35102015958954,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": -50,
        "score": 42.729217062398526
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 27.35102015958954
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      70.81973083659956,
      79.10365039256962
     ],
     "I": [
      6.5873945818039585,
      11.663246480964204
     ],
     "capability": [
      39.54192140892927,
      44.55782071854231
     ],
     "composite": null,
     "seed": 81
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 76.81388603317517,
     "capability": 42.63530955244829,
     "intelligence": 8.45673307172142
    },
    "per_type": {
     "choice": {
      "accuracy": 0.43066666666666664,
      "calibration": 84.2336491718195,
      "cc": 26.462427524445555,
      "cc_by_type": {
       "choice": 26.462427524445555
      },
      "cc_tiered": 25.370199215164263,
      "cc_tiered_open": 23.403612503766364,
      "cc_tiered_sealed": 27.336785926562165,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 26.462427524445555
     },
     "noul": {
      "accuracy": 0.25066666666666665,
      "calibration": 74.8703517596191,
      "cc": -49.86666666666667,
      "cc_by_type": {
       "noul": -49.86666666666667
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.368,
      "calibration": 71.33765716808692,
      "cc": 0.9290025644768996,
      "cc_by_type": {
       "score": 0.9290025644768996
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0.9290025644768996
     }
    },
    "raw_on_A": {
     "calibration": 75.03038816750606,
     "capability": 42.19031445358924,
     "intelligence": 9.350240739672419
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.6039581928545337,
    "Emphasis on Accuracy 60:20:20": 0.37780040258220743,
    "Emphasis on Cost 20:20:60": 0.8550985238185931,
    "Emphasis on Speed 20:60:20": 0.8689040584906251,
    "Intelligence only": 0.24191781959574568
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Nandakishor M",
   "axes": {
    "calibration": 62.93608607777928,
    "cost": 68.03119990679988,
    "intelligence": 3.0532964296406795,
    "speed": 89.16338093060068
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.20193798693071482,
      "ece_score": 59.61240261385703,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 59.61240261385703,
      "score_before_time_limit": 59.61240261385703,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.6799999999999999,
      "brier": 0.28375891613054305,
      "ece": 0.17069524022939242,
      "n": 375,
      "score": 65.86095195412152,
      "score_before_time_limit": 65.86095195412152,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.728,
      "interval90_width": 0.4346666666666666,
      "n": 375,
      "nrps": 0.195768816097399,
      "score": 63.33490366535929,
      "score_before_time_limit": 63.33490366535929,
      "time_limit_failed_share": 0,
      "top_ece": 0.2687665552977076
     }
    },
    "score": 62.93608607777928
   },
   "capability": 32.99469125370998,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 0.8B active parameters; throughput 100000 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,885,865 input + 0 output tokens over 1,500 decisions = 1923.91 tokens/decision. GPU-time cost USD 0.0116312574801587/1,000; old list-price cost USD 0.0192391/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0116312574801587/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011631257480158733
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Vega 0.8B",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX A6000 48 GB pod (Kansas City, US), offline after setup (egress cut and proven), reviewed vegaml==0.6.1 wheel, backbone bf16 / engine fp32, context 73,728 (longer states cut on the right), reference PyTorch path for linear attention (no flash-linear-attention / causal-conv1d)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 4.066515604051393,
    "I_sealed": 2.040077255229966,
    "base": 3.0532964296406795,
    "excess": -8.046806232518286,
    "gap": 2.0264383488214266,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 12.19954681215418,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 14.378378378378391,
       "hard": 9.405473419314262,
       "judge": 16.94133157547792,
       "standard": 9.585600669736293
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 14.28571428571428,
       "hard": -73.91304347826086,
       "judge": -50,
       "standard": -55.55555555555556
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.625135136625836,
       "hard": -0.915170580118807,
       "judge": -9.170003719084164,
       "standard": -19.439472746148546
      }
     },
     "sealed|choice": {
      "cc": 6.1202317656898995,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": -8.874752320367087,
       "hard": 18.874354182697587,
       "judge": 3.2012581807112888,
       "standard": -7.512060647829066
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": -23.076923076923073,
       "hard": -69.6969696969697,
       "judge": -58.33333333333333,
       "standard": -78.94736842105263
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": -0.09009355832638732,
       "hard": -3.84788500990465,
       "judge": -4.989943518704232,
       "standard": -15.600158434735768
      }
     }
    }
   },
   "jevbench_score": 0.040388417880414994,
   "key": "vegaml-0.8b",
   "last_measured_on": "2026-10-10",
   "licence": "vegaml package Apache-2.0; HF checkpoint repo declares no licence field (implied, not declared); backbone Qwen3.5 Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nandakishorm/vega-08b-public-intents@73a76a4f6c4fe672e0daa2d45ddc355afc5bc9dd",
   "noul_decisive": {
    "acc_among_decisive": 0.6333333333333333,
    "decisive": 120,
    "decisive_rate": 0.32,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 62.93608607777928,
     "cost": 61.47445744011434,
     "intelligence": 3.0532964296406795,
     "speed": 89.45598874418484
    },
    "board_release": "v1.6.43",
    "capability": 32.99469125370998,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 124,
    "capability_rank": 153,
    "composite_ci95": {
     "A": [
      0.0035937243990094452,
      0.23534462793689395
     ],
     "B": [
      0.00231035635955886,
      0.16355006319549895
     ],
     "C": [
      0.002495641943756559,
      0.16343376940062082
     ]
    },
    "jevbench_score": 0.04022166962784317,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 183,
    "rank": 227,
    "ranks": {
     "A": 227,
     "B": 227,
     "C": 227,
     "capability": 220
    },
    "scores": {
     "A": 0.04022166962784317,
     "B": 0.0266984340300977,
     "C": 0.027931715019335536
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.3191188830882311,
     "p50_s_raw": 0.08455944154411554,
     "p95_s_adjusted": 0.35518009765073655,
     "p95_s_raw": 0.10259004882536829
    },
    "views": {
     "Balanced 33:33:33": 0.03151658754618596,
     "Emphasis on Accuracy 60:20:20": 0.018460810494659353,
     "Emphasis on Cost 20:20:60": 0.048117449859240324,
     "Emphasis on Speed 20:60:20": 0.04941520456803552,
     "Intelligence only": 0.011385887829608154
    }
   },
   "rank": 269,
   "ranked": true,
   "ranks": {
    "A": 269,
    "B": 269,
    "C": 269,
    "capability": 261
   },
   "repo": "https://huggingface.co/nandakishorm/vega-08b-public-intents",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.040388417880414994,
    "B": 0.02675709633382359,
    "C": 0.02804751241695486
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3234992638230324,
    "p50_s_raw": 0.08674963191151619,
    "p95_s_adjusted": 0.3747905312106013,
    "p95_s_raw": 0.11239526560530065
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.27586206896551724,
       "cc": -19.36556571427138,
       "cc_by_type": {
        "choice": 25,
        "noul": -66.66666666666667,
        "score": -16.43003047614746
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 0
      },
      "calibration_gen": {
       "accuracy": 0.19642857142857142,
       "cc": -24.988246708550232,
       "cc_by_type": {
        "choice": -4.813664596273289,
        "noul": -76.92307692307692,
        "score": 6.772001393699512
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.15306122448979592,
       "cc": -36.524602578293646,
       "cc_by_type": {
        "choice": -3.7037037037037055,
        "noul": -100,
        "score": -5.870104031177226
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 0
      },
      "fairness": {
       "accuracy": 0.2222222222222222,
       "cc": -26.181811039531453,
       "cc_by_type": {
        "choice": -5.882352941176475,
        "noul": -100,
        "score": 27.336919822582118
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 0
      },
      "jaggedness": {
       "accuracy": 0.2864864864864865,
       "cc": -17.791678833140427,
       "cc_by_type": {
        "choice": 4.12227883279296,
        "noul": -23.076923076923073,
        "score": -34.42039225529116
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.22348484848484848,
       "cc": -10.709273113952321,
       "cc_by_type": {
        "choice": -3.111936832326967,
        "noul": -47.945205479452056,
        "score": 18.929322969922058
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 0
      },
      "long_state": {
       "accuracy": 0.1724137931034483,
       "cc": -34.941907856494,
       "cc_by_type": {
        "choice": 4.761904761904764,
        "noul": -81.81818181818181,
        "score": -27.769446513204944
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.41935483870967744,
       "cc": -14.749178737264756,
       "cc_by_type": {
        "choice": 7.692307692307688,
        "noul": 25,
        "score": -76.93984390410196
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.2634920634920635,
       "cc": -18.949056458195674,
       "cc_by_type": {
        "choice": 12.225705329153609,
        "noul": -67.81609195402298,
        "score": -1.2567827497176554
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 0
      },
      "negation_pair_gen": {
       "accuracy": 0.19642857142857142,
       "cc": -42.402473689854006,
       "cc_by_type": {
        "choice": -9.523809523809524,
        "noul": -60,
        "score": -57.6836115457525
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.3157894736842105,
       "cc": 13.42952275249723,
       "cc_by_type": {
        "choice": 13.42952275249723
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 13.42952275249723
      },
      "perturb_pair": {
       "accuracy": 0.45,
       "cc": 6.179349639681301,
       "cc_by_type": {
        "choice": 29.22673656618611,
        "noul": -14.28571428571429,
        "score": 3.5970266385720873
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 6.179349639681301
      },
      "usecase": {
       "accuracy": 0.29535864978902954,
       "cc": -19.00704443525642,
       "cc_by_type": {
        "choice": 21.37213335902873,
        "noul": -71.9298245614035,
        "score": -6.46344210339449
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.45454545454545453,
       "cc": -7.714580910237068,
       "cc_by_type": {
        "choice": -6.666666666666665,
        "noul": 0,
        "score": -16.47707606404454
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.25,
       "cc": -38.135593220338976,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.2916666666666667,
       "cc": -23.813705445205983,
       "cc_by_type": {
        "choice": 5.325443786982243,
        "noul": -100,
        "score": 23.23343987739981
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 0
      },
      "de": {
       "accuracy": 0.26666666666666666,
       "cc": -9.48435284148812,
       "cc_by_type": {
        "choice": 13.095956547978272,
        "noul": -33.333333333333336,
        "score": -8.215681739109293
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 0
      },
      "el": {
       "accuracy": 0.4444444444444444,
       "cc": -38.08299402731068,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": -60,
        "score": -94.00801822651034
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.2529043789097408,
       "cc": -19.4845221445307,
       "cc_by_type": {
        "choice": 6.449803701626394,
        "noul": -60.902255639097746,
        "score": -4.00111449612075
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 0
      },
      "es": {
       "accuracy": 0.35714285714285715,
       "cc": -3.020542459003753,
       "cc_by_type": {
        "choice": -17.81818181818182,
        "noul": 9.090909090909083,
        "score": -0.3343546497385219
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "fi": {
       "accuracy": 0.4444444444444444,
       "cc": 1.6431924882629083,
       "cc_by_type": {
        "choice": 36.61971830985915,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 1.6431924882629083
      },
      "fr": {
       "accuracy": 0.06666666666666667,
       "cc": -42.623688574546726,
       "cc_by_type": {
        "choice": -26.93935119887165,
        "noul": -100,
        "score": -0.9317145247685232
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "hi": {
       "accuracy": 0.2,
       "cc": -28.83602734828803,
       "cc_by_type": {
        "choice": -11.801242236024846,
        "noul": -40,
        "score": -34.70683980883924
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0
      },
      "id": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": -100
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.5294117647058824,
       "cc": 5.113250586139728,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": 0,
        "score": -33.88450776203216
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 5.113250586139728
      },
      "ja": {
       "accuracy": 0.21875,
       "cc": -21.637722673747195,
       "cc_by_type": {
        "choice": 2.4390243902439046,
        "noul": -71.42857142857143,
        "score": 4.076379017085941
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.18181818181818182,
       "cc": -20.186550397257573,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 0,
        "score": -62.1989954540678
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 0
      },
      "mixed": {
       "accuracy": 0.2727272727272727,
       "cc": -1.2255967997920887,
       "cc_by_type": {
        "choice": -9.890109890109892,
        "noul": -50,
        "score": 56.213319490733625
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "nl": {
       "accuracy": 0.07692307692307693,
       "cc": -33.67879710336313,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": -100,
        "score": -0.7578397780559687
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0,
       "cc": -65.57377049180329,
       "cc_by_type": {
        "choice": -31.14754098360656,
        "noul": -100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.275,
       "cc": -16.97531167960543,
       "cc_by_type": {
        "choice": 29.164844774814167,
        "noul": -80,
        "score": -0.0907798136304594
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 0
      },
      "pt": {
       "accuracy": 0.5,
       "cc": 21.729761287554258,
       "cc_by_type": {
        "choice": 81.90954773869345,
        "noul": -66.66666666666667,
        "score": 49.946402790636014
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 21.729761287554258
      },
      "sv": {
       "accuracy": 0.25,
       "cc": -8.670532037806744,
       "cc_by_type": {
        "choice": 11.111111111111109,
        "noul": -100,
        "score": 62.87729277546865
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.18181818181818182,
       "cc": -58.40033972849334,
       "cc_by_type": {
        "choice": -5.26315789473684,
        "noul": -100,
        "score": -69.93786129074316
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "uk": {
       "accuracy": 0.18181818181818182,
       "cc": -43.03199883446496,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": -42.13947476426443
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.5,
       "cc": 18.615495552140683,
       "cc_by_type": {
        "choice": 35.13513513513514,
        "noul": -33.333333333333336,
        "score": 54.04468485462024
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 18.615495552140683
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.2866666666666667,
       "cc": 7.583347728450512,
       "cc_by_type": {
        "choice": 7.583347728450512
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 7.583347728450512
      },
      "noul": {
       "accuracy": 0.20266666666666666,
       "cc": -59.46666666666667,
       "cc_by_type": {
        "noul": -59.46666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.264,
       "cc": -4.914138326410811,
       "cc_by_type": {
        "score": -4.914138326410811
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 0
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.25,
       "cc": -42.23115369530359,
       "cc_by_type": {
        "choice": 11.037891268533771,
        "noul": -100,
        "score": -37.73135235444454
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.4166666666666667,
       "cc": 10.053783668961861,
       "cc_by_type": {
        "choice": 21.440261865793783,
        "noul": -19.999999999999996,
        "score": 28.721089141091795
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 10.053783668961861
      },
      "customer_support": {
       "accuracy": 0.25,
       "cc": -14.695878083746338,
       "cc_by_type": {
        "choice": -12.90322580645161,
        "noul": -50,
        "score": 18.815591555212595
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.07692307692307693,
       "cc": -52.52931897614329,
       "cc_by_type": {
        "choice": -20.481927710843376,
        "noul": -100,
        "score": -37.10602921758648
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.1111111111111111,
       "cc": -38.90478330755122,
       "cc_by_type": {
        "choice": 6.832298136645961,
        "noul": -100,
        "score": -23.546648059299624
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.3793103448275862,
       "cc": -13.328851245398967,
       "cc_by_type": {
        "choice": 31.623931623931618,
        "noul": -60,
        "score": -11.61048536012852
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.34615384615384615,
       "cc": -18.503379574597428,
       "cc_by_type": {
        "choice": 47.82608695652174,
        "noul": -75,
        "score": -28.33622568031402
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "gaming": {
       "accuracy": 0.27586206896551724,
       "cc": -25.77506638432793,
       "cc_by_type": {
        "choice": -6.9306930693069315,
        "noul": -71.42857142857143,
        "score": 1.0340653448945747
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.14285714285714285,
       "cc": -30.119048798099303,
       "cc_by_type": {
        "choice": -11.11111111111111,
        "noul": -100,
        "score": 20.7539647168132
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.17391304347826086,
       "cc": -34.69531570651282,
       "cc_by_type": {
        "choice": -0.7633587786259586,
        "noul": -100,
        "score": -3.3225883409125245
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.48,
       "cc": 8.261797778580346,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": -25,
        "score": -17.694281461006934
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 8.261797778580346
      },
      "legal_compliance": {
       "accuracy": 0.23076923076923078,
       "cc": -31.134132674360444,
       "cc_by_type": {
        "choice": 18.032786885245905,
        "noul": -100,
        "score": -11.435184908327244
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.3103448275862069,
       "cc": -9.67272641203819,
       "cc_by_type": {
        "choice": 17.431192660550458,
        "noul": -75,
        "score": 28.550628103334972
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.29333333333333333,
       "cc": 2.0858721093715658,
       "cc_by_type": {
        "choice": 1.9029750737067974,
        "noul": -18.181818181818176,
        "score": 22.536459436226075
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 2.0858721093715658
      },
      "moderation_trust_safety": {
       "accuracy": 0.3125,
       "cc": -22.66548316017918,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": -100,
        "score": -4.8385547436954335
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 0
      },
      "recruiting": {
       "accuracy": 0.38095238095238093,
       "cc": 1.0762044705122094,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": -66.66666666666667,
        "score": 0.046033847047532994
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 1.0762044705122094
      },
      "risk_assessment": {
       "accuracy": 0.3076923076923077,
       "cc": -21.111851703106247,
       "cc_by_type": {
        "choice": 24.528301886792452,
        "noul": -100,
        "score": 12.136143003888812
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "scientific_discovery": {
       "accuracy": 0.1875,
       "cc": -22.40823864613203,
       "cc_by_type": {
        "choice": 4.382470119521911,
        "noul": -71.42857142857143,
        "score": -0.17861462934656291
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.2727272727272727,
       "cc": -35.0856709518421,
       "cc_by_type": {
        "choice": 14.08114558472553,
        "noul": -66.66666666666667,
        "score": -52.671491773585146
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 0
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      57.40975035799149,
      66.49290908060016
     ],
     "I": [
      1.3348314800373893,
      5.689355834192679
     ],
     "capability": [
      29.933677735120224,
      35.37515714416046
     ],
     "composite": null,
     "seed": 29
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 62.93608607777928,
     "capability": 32.99469125370998,
     "intelligence": 3.0532964296406795
    },
    "per_type": {
     "choice": {
      "accuracy": 0.2866666666666667,
      "calibration": 59.61240261385703,
      "cc": 7.583347728450512,
      "cc_by_type": {
       "choice": 7.583347728450512
      },
      "cc_tiered": 9.15988928892204,
      "cc_tiered_open": 12.19954681215418,
      "cc_tiered_sealed": 6.1202317656898995,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 7.583347728450512
     },
     "noul": {
      "accuracy": 0.20266666666666666,
      "calibration": 65.86095195412152,
      "cc": -59.46666666666667,
      "cc_by_type": {
       "noul": -59.46666666666667
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.264,
      "calibration": 63.33490366535929,
      "cc": -4.914138326410811,
      "cc_by_type": {
       "score": -4.914138326410811
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 59.68341710343884,
     "capability": 30.94517056072779,
     "intelligence": 2.206924018016739
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.03165312091858969,
    "Emphasis on Accuracy 60:20:20": 0.018488838664457323,
    "Emphasis on Cost 20:20:60": 0.04870400001403727,
    "Emphasis on Speed 20:60:20": 0.04960685510201345,
    "Intelligence only": 0.011385887829608154
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Nandakishor M",
   "axes": {
    "calibration": 57.46108428162757,
    "cost": 61.702748578045096,
    "intelligence": 15.69626515886651,
    "speed": 83.93992701495202
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.2574125103434302,
      "ece_score": 48.517497931313954,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 47.80590796165468,
      "score_before_time_limit": 48.517497931313954,
      "time_limit_failed_share": 0.014666666666666666
     },
     "noul": {
      "abstention_rate": 0.30400000000000005,
      "brier": 0.2781281471262366,
      "ece": 0.2285300562531464,
      "n": 375,
      "score": 52.701365079389184,
      "score_before_time_limit": 54.293988749370726,
      "time_limit_failed_share": 0.029333333333333333
     },
     "score": {
      "interval90_coverage": 0.8346666666666667,
      "interval90_width": 0.49288888888888904,
      "n": 375,
      "nrps": 0.16226214991492005,
      "score": 71.87597980383885,
      "score_before_time_limit": 73.24318594141187,
      "time_limit_failed_share": 0.018666666666666668,
      "top_ece": 0.18643706562842138
     }
    },
    "score": 57.46108428162757
   },
   "capability": 36.57867472024704,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 2,885,865 input + 0 output tokens over 1,500 decisions = 1923.91 tokens/decision. GPU-time cost USD 0.0189049288584457/1,000; old list-price cost USD 0.0577173/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0189049288584457/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.018904928858445722
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 29,
    "n_over_limit_answered_ok": 29,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Vega 4B",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium RTX A6000 48 GB pod (Des Moines, US), offline after setup (egress cut and proven), reviewed vegaml==0.6.1 wheel, backbone bf16 / engine fp32, context 73,728 (longer states cut on the right), reference PyTorch path for linear attention (no flash-linear-attention / causal-conv1d)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 21.771877584149628,
    "I_sealed": 9.620652733583393,
    "base": 15.69626515886651,
    "excess": 2.077980269226522,
    "gap": 12.151224850566235,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 41.396463388934684,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 45.51351351351353,
       "hard": 28.279333123623786,
       "judge": 48.58272907053395,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 9.665976535541756,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 57.14285714285714,
       "hard": 30.434782608695656,
       "judge": -19.999999999999996,
       "standard": -11.111111111111116
      }
     },
     "open|score": {
      "cc": 14.253192827972441,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 36.64297240164319,
       "hard": 6.501670939484594,
       "judge": 20.791162375101624,
       "standard": 8.754392497418984
      }
     },
     "sealed|choice": {
      "cc": 24.831311180753584,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 32.4225675252894,
       "hard": 22.337034187094638,
       "judge": 22.378367409060942,
       "standard": 29.70365265334253
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 7.692307692307687,
       "hard": -28.787878787878785,
       "judge": 8.333333333333325,
       "standard": -12.28070175438597
      }
     },
     "sealed|score": {
      "cc": 4.030647019996595,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 23.787017462891814,
       "hard": -4.6189235136853135,
       "judge": 10.43509835324301,
       "standard": 1.8449258660431789
      }
     }
    }
   },
   "jevbench_score": 3.608782255985968,
   "key": "vegaml-4b",
   "last_measured_on": "2026-10-10",
   "licence": "vegaml package Apache-2.0; HF checkpoint repo declares no licence field (implied, not declared); backbone Qwen3.5 Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "nandakishorm/vega-08b-public-intents@73a76a4f6c4fe672e0daa2d45ddc355afc5bc9dd",
   "noul_decisive": {
    "acc_among_decisive": 0.696,
    "decisive": 250,
    "decisive_rate": 0.6868131868131868,
    "n": 375,
    "supported": true,
    "valid": 364
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 58.68489087403218,
     "cost": 47.16081979852447,
     "intelligence": 16.689214224692094,
     "speed": 84.39386641436894
    },
    "board_release": "v1.6.43",
    "capability": 37.687052549362136,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 118,
    "capability_rank": 146,
    "composite_ci95": {
     "A": [
      1.5678216817255275,
      6.594029572003121
     ],
     "B": [
      1.2095817426628035,
      5.54602239255017
     ],
     "C": [
      1.0887650567538385,
      4.579187202779947
     ]
    },
    "jevbench_score": 3.603902446043355,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 148,
    "rank": 188,
    "ranks": {
     "A": 188,
     "B": 191,
     "C": 189,
     "capability": 210
    },
    "scores": {
     "A": 3.603902446043355,
     "B": 2.9164270830443,
     "C": 2.502710031974551
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.5304726889356971,
     "p50_s_raw": 0.19023634446784854,
     "p95_s_adjusted": 0.6854098306968808,
     "p95_s_raw": 0.2677049153484404
    },
    "views": {
     "Balanced 33:33:33": 3.198325647386112,
     "Emphasis on Accuracy 60:20:20": 2.3288005981136943,
     "Emphasis on Cost 20:20:60": 3.6607417516659844,
     "Emphasis on Speed 20:60:20": 4.247790339721065,
     "Intelligence only": 1.654208859284027
    }
   },
   "rank": 232,
   "ranked": true,
   "ranks": {
    "A": 232,
    "B": 233,
    "C": 232,
    "capability": 254
   },
   "repo": "https://huggingface.co/nandakishorm/vega-08b-public-intents",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.608782255985968,
    "B": 2.84919590690567,
    "C": 2.506098788879145
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5442217165604234,
    "p50_s_raw": 0.1971108582802117,
    "p95_s_adjusted": 0.7417053825221955,
    "p95_s_raw": 0.29585269126109776
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.5862068965517241,
       "cc": 32.10254092129096,
       "cc_by_type": {
        "choice": 50,
        "noul": 0,
        "score": 46.30762276387288
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 32.10254092129096
      },
      "calibration_gen": {
       "accuracy": 0.3482142857142857,
       "cc": -7.764599070714067,
       "cc_by_type": {
        "choice": 16.14906832298137,
        "noul": -38.46153846153846,
        "score": -0.9813270735851143
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 0
      },
      "composite_gen": {
       "accuracy": 0.35714285714285715,
       "cc": 4.1474482072018475,
       "cc_by_type": {
        "choice": 7.407407407407411,
        "noul": -7.14285714285714,
        "score": 12.177794357055271
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 4.1474482072018475
      },
      "fairness": {
       "accuracy": 0.7037037037037037,
       "cc": 46.819844941220424,
       "cc_by_type": {
        "choice": 64.70588235294117,
        "noul": 0,
        "score": 75.75365247072008
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 46.819844941220424
      },
      "jaggedness": {
       "accuracy": 0.34054054054054056,
       "cc": -9.581651647472881,
       "cc_by_type": {
        "choice": 19.407132931912926,
        "noul": -28.205128205128204,
        "score": -19.946959669203366
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 0
      },
      "llm_router": {
       "accuracy": 0.5303030303030303,
       "cc": 35.11456970853259,
       "cc_by_type": {
        "choice": 34.9744542498839,
        "noul": 28.767123287671236,
        "score": 41.602131588042646
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 35.11456970853259
      },
      "long_state": {
       "accuracy": 0,
       "cc": -110.31746031746032,
       "cc_by_type": {
        "choice": -30.952380952380945,
        "noul": -100,
        "score": -200
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.3548387096774194,
       "cc": -22.26927167429365,
       "cc_by_type": {
        "choice": 35.38461538461538,
        "noul": -75,
        "score": -27.192430407496325
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.4603174603174603,
       "cc": 15.9575022436564,
       "cc_by_type": {
        "choice": 35.512763098970005,
        "noul": -1.1494252873563204,
        "score": 13.509168919355519
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 15.9575022436564
      },
      "negation_pair_gen": {
       "accuracy": 0.32142857142857145,
       "cc": -18.937881655648187,
       "cc_by_type": {
        "choice": -9.523809523809524,
        "noul": 0,
        "score": -47.28983544313503
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 0
      },
      "option_order": {
       "accuracy": 0.5614035087719298,
       "cc": 44.50610432852386,
       "cc_by_type": {
        "choice": 44.50610432852386
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 44.50610432852386
      },
      "perturb_pair": {
       "accuracy": 0.5333333333333333,
       "cc": 41.1419841739421,
       "cc_by_type": {
        "choice": 25.29488859764089,
        "noul": 42.85714285714286,
        "score": 55.273921067042544
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 41.1419841739421
      },
      "usecase": {
       "accuracy": 0.379746835443038,
       "cc": -1.691456792175881,
       "cc_by_type": {
        "choice": 28.30988629793796,
        "noul": -22.807017543859654,
        "score": -10.577239130605953
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 0
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5454545454545454,
       "cc": -14.905334686220934,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": -50,
        "score": -68.04933739199615
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.375,
       "cc": -4.802259887005647,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 0
      },
      "da": {
       "accuracy": 0.4583333333333333,
       "cc": 22.143800617963134,
       "cc_by_type": {
        "choice": 64.49704142011835,
        "noul": 19.999999999999996,
        "score": -18.065639566228953
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 22.143800617963134
      },
      "de": {
       "accuracy": 0.4444444444444444,
       "cc": 3.742298256027752,
       "cc_by_type": {
        "choice": 29.390464695232346,
        "noul": -33.333333333333336,
        "score": 15.169763406184245
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 3.742298256027752
      },
      "el": {
       "accuracy": 0.5,
       "cc": -7.267096986083452,
       "cc_by_type": {
        "choice": 51.80722891566265,
        "noul": -60,
        "score": -13.608519873913005
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 0
      },
      "en": {
       "accuracy": 0.42001787310098304,
       "cc": 7.619934970948897,
       "cc_by_type": {
        "choice": 25.96747055524391,
        "noul": -8.270676691729328,
        "score": 5.163011049332111
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 7.619934970948897
      },
      "es": {
       "accuracy": 0.5357142857142857,
       "cc": 23.735811235182705,
       "cc_by_type": {
        "choice": 47.63636363636363,
        "noul": 9.090909090909083,
        "score": 14.480160978275391
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 23.735811235182705
      },
      "fi": {
       "accuracy": 0.2222222222222222,
       "cc": -19.483568075117372,
       "cc_by_type": {
        "choice": -5.63380281690141,
        "noul": -33.333333333333336
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 0
      },
      "fr": {
       "accuracy": 0.5333333333333333,
       "cc": 9.81849210967231,
       "cc_by_type": {
        "choice": 49.22425952045134,
        "noul": -50,
        "score": 30.231216808565588
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 9.81849210967231
      },
      "hi": {
       "accuracy": 0.36666666666666664,
       "cc": -2.9723800154374516,
       "cc_by_type": {
        "choice": 37.888198757763966,
        "noul": -40,
        "score": -6.805338804076322
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 0
      },
      "id": {
       "accuracy": 0.16666666666666666,
       "cc": -14.516129032258062,
       "cc_by_type": {
        "choice": -29.032258064516125,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.5882352941176471,
       "cc": 41.890588956237984,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 50,
        "score": 1.059637108488276
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 41.890588956237984
      },
      "ja": {
       "accuracy": 0.34375,
       "cc": 6.462836027337623,
       "cc_by_type": {
        "choice": -5.691056910569103,
        "noul": -14.28571428571429,
        "score": 39.36527927829626
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 6.462836027337623
      },
      "ko": {
       "accuracy": 0.36363636363636365,
       "cc": 18.2762217539951,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 0,
        "score": 20.402435753788563
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 18.2762217539951
      },
      "mixed": {
       "accuracy": 0.7272727272727273,
       "cc": 59.83071388979721,
       "cc_by_type": {
        "choice": 56.04395604395604,
        "noul": 100,
        "score": 23.448185625435602
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 59.83071388979721
      },
      "nl": {
       "accuracy": 0.15384615384615385,
       "cc": -18.26355956150215,
       "cc_by_type": {
        "choice": -25.34818941504178,
        "noul": -66.66666666666667,
        "score": 37.224177397202
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.7142857142857143,
       "cc": 67.21311475409837,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 67.21311475409837
      },
      "pl": {
       "accuracy": 0.525,
       "cc": 43.18418848568871,
       "cc_by_type": {
        "choice": 68.51770878880629,
        "noul": 60.00000000000001,
        "score": 1.0348566682598204
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 43.18418848568871
      },
      "pt": {
       "accuracy": 0.4444444444444444,
       "cc": 15.083741051175762,
       "cc_by_type": {
        "choice": 27.638190954773865,
        "noul": -33.333333333333336,
        "score": 50.94636553208676
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 15.083741051175762
      },
      "sv": {
       "accuracy": 0.125,
       "cc": -24.998495011569247,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": -50,
        "score": 8.337848298625605
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.6363636363636364,
       "cc": 20.74230849140807,
       "cc_by_type": {
        "choice": 47.368421052631575,
        "noul": 100,
        "score": -85.14149557840736
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 20.74230849140807
      },
      "uk": {
       "accuracy": 0.18181818181818182,
       "cc": -41.58877432033782,
       "cc_by_type": {
        "choice": 13.043478260869565,
        "noul": -100,
        "score": -37.80980122188302
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.75,
       "cc": 65.40226528868891,
       "cc_by_type": {
        "choice": 67.56756756756756,
        "noul": 33.33333333333333,
        "score": 95.30589496516582
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 65.40226528868891
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.444,
       "cc": 27.966833650025915,
       "cc_by_type": {
        "choice": 27.966833650025915
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 27.966833650025915
      },
      "noul": {
       "accuracy": 0.464,
       "cc": -7.199999999999996,
       "cc_by_type": {
        "noul": -7.199999999999996
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 0
      },
      "score": {
       "accuracy": 0.35733333333333334,
       "cc": 6.1512696164423115,
       "cc_by_type": {
        "score": 6.1512696164423115
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 6.1512696164423115
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.3333333333333333,
       "cc": -35.1534607494542,
       "cc_by_type": {
        "choice": 20.922570016474467,
        "noul": -100,
        "score": -26.382952264837066
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.5416666666666666,
       "cc": 15.916918386842573,
       "cc_by_type": {
        "choice": 41.08019639934533,
        "noul": 19.999999999999996,
        "score": -13.329441238817608
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 15.916918386842573
      },
      "customer_support": {
       "accuracy": 0.45,
       "cc": 8.926626674691754,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 25,
        "score": -33.703990943666675
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 8.926626674691754
      },
      "demand_forecasting": {
       "accuracy": 0.23076923076923078,
       "cc": -37.51418716919469,
       "cc_by_type": {
        "choice": 3.614457831325295,
        "noul": -100,
        "score": -16.15701933890936
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.2222222222222222,
       "cc": -13.643840746791454,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -60,
        "score": -6.397360749691128
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.27586206896551724,
       "cc": -6.646648832243309,
       "cc_by_type": {
        "choice": 5.9829059829059785,
        "noul": -19.999999999999996,
        "score": -5.922852479635909
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6538461538461539,
       "cc": 30.68478198913924,
       "cc_by_type": {
        "choice": 86.95652173913045,
        "noul": 25,
        "score": -19.902175771712738
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 30.68478198913924
      },
      "gaming": {
       "accuracy": 0.27586206896551724,
       "cc": -8.539638885229605,
       "cc_by_type": {
        "choice": -6.9306930693069315,
        "noul": -42.85714285714286,
        "score": 24.16891927076098
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.21428571428571427,
       "cc": -26.327496496464544,
       "cc_by_type": {
        "choice": 4.761904761904763,
        "noul": -60,
        "score": -23.744394251298395
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 0
      },
      "knowledge_graphs": {
       "accuracy": 0.21739130434782608,
       "cc": -7.707530000989269,
       "cc_by_type": {
        "choice": -0.7633587786259586,
        "noul": -19.999999999999996,
        "score": -2.359231224341851
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.52,
       "cc": 19.542316672389997,
       "cc_by_type": {
        "choice": 51.21951219512196,
        "noul": 25,
        "score": -17.592562177951955
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 19.542316672389997
      },
      "legal_compliance": {
       "accuracy": 0.23076923076923078,
       "cc": -17.21253914360014,
       "cc_by_type": {
        "choice": 18.032786885245905,
        "noul": -100,
        "score": 30.32959568395368
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.41379310344827586,
       "cc": -0.8829038829667036,
       "cc_by_type": {
        "choice": 35.77981651376147,
        "noul": -50,
        "score": 11.571471837338422
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 0
      },
      "model_routing": {
       "accuracy": 0.5,
       "cc": 24.299647249468666,
       "cc_by_type": {
        "choice": 42.106673813990895,
        "noul": 22.72727272727273,
        "score": 8.064995207142367
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 24.299647249468666
      },
      "moderation_trust_safety": {
       "accuracy": 0.5,
       "cc": 21.15669006850672,
       "cc_by_type": {
        "choice": 21.052631578947366,
        "noul": 19.999999999999996,
        "score": 22.417438626572796
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 21.15669006850672
      },
      "recruiting": {
       "accuracy": 0.42857142857142855,
       "cc": 4.28970309222692,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": -33.333333333333336,
        "score": -23.646803621141665
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 4.28970309222692
      },
      "risk_assessment": {
       "accuracy": 0.5384615384615384,
       "cc": 13.524055414252595,
       "cc_by_type": {
        "choice": 43.39622641509434,
        "noul": 33.33333333333333,
        "score": -36.15739350566989
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 13.524055414252595
      },
      "scientific_discovery": {
       "accuracy": 0.21875,
       "cc": -20.974690029781613,
       "cc_by_type": {
        "choice": 20.31872509960159,
        "noul": -71.42857142857143,
        "score": -11.814223760374997
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 0
      },
      "search_retrieval": {
       "accuracy": 0.5,
       "cc": 2.15502085448014,
       "cc_by_type": {
        "choice": 57.04057279236276,
        "noul": -33.333333333333336,
        "score": -17.242176895589
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 2.15502085448014
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      52.22926067223202,
      61.459644474882964
     ],
     "I": [
      11.222571232188471,
      20.06206658675151
     ],
     "capability": [
      32.61433592571537,
      39.87642689527607
     ],
     "composite": null,
     "seed": 30
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 57.46108428162757,
     "capability": 36.57867472024704,
     "intelligence": 15.69626515886651
    },
    "per_type": {
     "choice": {
      "accuracy": 0.444,
      "calibration": 47.80590796165468,
      "cc": 27.966833650025915,
      "cc_by_type": {
       "choice": 27.966833650025915
      },
      "cc_tiered": 33.11388728484413,
      "cc_tiered_open": 41.396463388934684,
      "cc_tiered_sealed": 24.831311180753584,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 27.966833650025915
     },
     "noul": {
      "accuracy": 0.464,
      "calibration": 52.701365079389184,
      "cc": -7.199999999999996,
      "cc_by_type": {
       "noul": -7.199999999999996
      },
      "cc_tiered": 4.832988267770878,
      "cc_tiered_open": 9.665976535541756,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.35733333333333334,
      "calibration": 71.87597980383885,
      "cc": 6.1512696164423115,
      "cc_by_type": {
       "score": 6.1512696164423115
      },
      "cc_tiered": 9.141919923984517,
      "cc_tiered_open": 14.253192827972441,
      "cc_tiered_sealed": 4.030647019996595,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 6.1512696164423115
     }
    },
    "raw_on_A": {
     "calibration": 59.42725004267636,
     "capability": 37.79076765897078,
     "intelligence": 16.15428527526519
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.014666666666666717,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.01866666666666672,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 3.2195262542833616,
    "Emphasis on Accuracy 60:20:20": 2.2474314003083697,
    "Emphasis on Cost 20:20:60": 3.965984846388119,
    "Emphasis on Speed 20:60:20": 4.260440199690533,
    "Intelligence only": 1.5468527415898499
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "scar-ai (Hugging Face)",
   "axes": {
    "calibration": 91.31679956749362,
    "cost": 55.930081512098035,
    "intelligence": 67.78675651753129,
    "speed": 88.01635603332511
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.04184131147541006,
      "ece_score": 91.631737704918,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 91.631737704918,
      "score_before_time_limit": 91.631737704918,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.1769436997319035,
      "brier": 0.07210259847184984,
      "ece": 0.047123056300268036,
      "n": 373,
      "score": 90.57538873994639,
      "score_before_time_limit": 90.57538873994639,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.978494623655914,
      "interval90_width": 0.3006272401433691,
      "n": 372,
      "nrps": 0.050566275277777725,
      "score": 91.7432722576165,
      "score_before_time_limit": 91.7432722576165,
      "time_limit_failed_share": 0,
      "top_ece": 0.05728413978494613
     }
    },
    "score": 91.31679956749362
   },
   "capability": 79.55177804251245,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 766,685 input + 84,061 output tokens over 1,485 decisions = 572.892929293 tokens/decision. GPU-time cost USD 0.0294440321271629/1,000; old list-price cost USD 0.216840202020202/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0294440321271629/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.2146718
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.029444032127162855
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Vera Large",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5991917808219176/h recorded by the runner. Attempt 2 measured 24 Oct 2026 08:36-08:53Z, about USD 0.74 GPU rental (approximation); teardown confirmed 08:53:47Z. Author server on 127.0.0.1:8200 in a --network none --read-only container; serial loopback driver. Read-only code, model, input and exported cache; empty credential variables, HF offline flags and author steering switches unset. Cut proof inside the container before held-out input: only loopback, no IPv4/IPv6 default route, DNS and TCP fail. Complete=true, deadline_hit=false, rc 0, measurement container 308 s (run cap 538 s), scored loop 155.156841 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked inside the container. Raw output sha256 468809896014afa52e577106142dbf1ff21d4d75ee7b55fe02d1f5993ee8fcea (receipt only; not independently inspected). Final node-shred verified; exact-id teardown confirms provider absent and guard closed. GPU rental is separate from the benchmark's estimated base-model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 72.66290812065732,
    "I_sealed": 62.91060491440525,
    "base": 67.78675651753129,
    "excess": -0.0872235641036383,
    "gap": 9.752303206252073,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 83.61812666623719,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 83.01352626612143,
       "judge": 88.13447593935399,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 69.94824016563147,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 82.6086956521739,
       "judge": 50,
       "standard": 66.66666666666667
      }
     },
     "open|score": {
      "cc": 64.4223575301033,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 68.83060606060607,
       "hard": 78.76416666666667,
       "judge": 60.168644067796606,
       "standard": 39.91518518518517
      }
     },
     "sealed|choice": {
      "cc": 79.73075848398948,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 92.37611181702668,
       "hard": 75.5406499363941,
       "judge": 77.08129285014691,
       "standard": 85.76249736342545
      }
     },
     "sealed|noul": {
      "cc": 49.20962825131636,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 75.86206896551724,
       "hard": 39.99999999999999,
       "judge": 44.61538461538461,
       "standard": 61.19402985074627
      }
     },
     "sealed|score": {
      "cc": 59.791428007909914,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 81.4221052631579,
       "hard": 55.378788732394376,
       "judge": 56.79755208333334,
       "standard": 62.2921818181818
      }
     }
    }
   },
   "jevbench_score": 72.80136805547387,
   "key": "vera-large",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 in the model card and vera-s1 0.1.6 package metadata; no standalone LICENSE file; public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "scar-ai/vera-large@cc5adb16238d9ebae887da71f994cd04963d251f",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9543973941368078,
    "decisive": 307,
    "decisive_rate": 0.8230563002680965,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 49,
   "ranked": true,
   "ranks": {
    "A": 49,
    "B": 39,
    "C": 40,
    "capability": 38
   },
   "repo": "https://huggingface.co/scar-ai/vera-large",
   "scores": {
    "A": 72.80136805547387,
    "B": 71.73995713899879,
    "C": 72.80136805547387
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3349350960000038,
    "p50_s_raw": 0.0924675480000019,
    "p95_s_adjusted": 0.4714153714000302,
    "p95_s_raw": 0.16070768570001512
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN SERVER: scar-ai/vera-large at cc5adb16238d9ebae887da71f994cd04963d251f with the author's vera-s1 0.1.6 package, installed from the repo's bundled wheel (sha256 94f5a09cf70e4480994230300963789841e0a7de3ff424b2df619fd76fe4de46) with --no-deps --no-index under a network-free image build; all members equal the PyPI 0.1.6 wheel's (different zip timestamps). Six author Python modules; the model repo's separate Python files and fp32 branch were not fetched. Author files are unchanged; DISCLOSED SERVING WRAPPER: a one-line wrapper around vera.load passes max_num_seqs=256 through the author's own **engine_kw API, then runs vera.serve as __main__, with --model /model --engine vllm --max-model-len 32768 --host 127.0.0.1 --port 8200. This caps the scheduler's maximum co-batched sequences, following the pre-sealed attempt-1 startup failure; it does not change the author's 32,768-token limit, prompt rendering or refusal set. vLLM 0.30.0 pooling runner (token_embed), bidirectional full attention (is_causal: false), causal Gated-DeltaNet layers, prefix caching and chunked prefill off, max_num_batched_tokens = max_model_len = 32768, author-default gpu_memory_utilization 0.9, fp32 option-pointer readout. One request at a time. Image: vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 plus the one wheel, derived image sha256:9ab6ff921642238843ef1daa85e6c05cf38c938cb7f776e4c84a12bcd6d6b61c. torch 2.13.0, transformers 5.17.0, tokenizers 0.23.2, safetensors 0.8.0. The author's export_backbone transformed the pinned backbone offline before any held-out input, then the cache was mounted read-only. Export verification: 850 bf16 backbone tensors, names/shapes/dtypes checked, 43 tensors byte-compared by sampling, every exported file hashed, verdict OK; not a byte comparison of every tensor. Weights: 25,628,527,110 parameters (25,624,600,064 bf16 backbone + 3,927,046 fp32 heads), 10 shards totalling 51,265,014,328 bytes, 16 data files in the pinned manifest (sha256 57d88387c7503633362b61eb9c17d05ea8608accf48774ab13cf68f51634cdde). The submitted code and serving wrapper were reviewed and cleared before the run; evaluator exposure and the device were cleared for both attempts. The source, runner, device selection change and serving wrapper changes were reviewed before the run; required safeguards were checked and the code and device cleared. DISCLOSED PATTERNS: dormant Hub downloads and Hub-capable loaders, weights-only torch.load fallbacks absent from the safetensors serving path, steering environment switches asserted unset, and the contained pre-input cache-writing export. No subprocess, outbound HTTP, telemetry, credential reads or request-body logging in the reviewed author package; offline flags and the network cut were checked inside the measurement container. FIRST MEASUREMENT of Vera Large in this intake. AUTHOR'S OWN SERVER on our rented hardware, vLLM pooling path with the disclosed max_num_seqs=256 wrapper; author files unchanged. NATIVE OUTPUTS: choice and score answers with probabilities, and noul P(true), from the author's readout and option pointer; probabilities rounded to 4 decimals; no generated explanation. AUTHOR EXPOSURE: training and contamination statements in the card are author claims; no independent overlap or contamination audit was run by us. The card's Decision Index 68.4 on a 5% sample and 139 ms median on an AMD MI300X are self-reported and do not transfer to this measurement. LICENCE NOTE: Apache-2.0 in the model card and package metadata; no standalone LICENSE file. ATTEMPT HISTORY: attempt 1 (pod window 24 Oct 2026 06:22-06:32Z, about USD 0.45) failed BEFORE any held-out item, at vLLM startup. CUDA-graph memory profiling tried to allocate a 78,920,024,064-byte (78.9 GB) throwaway cache of 512 sequences alongside about 51 GB of weights. pre-registration Amendment 3 documents the fix: max_num_seqs=256 through vera.load(**engine_kw), reducing that profiling allocation to 39,460,012,032 bytes; author files, the 32,768-token limit and the refusal set unchanged. The runner was rechecked, and code, evaluator exposure and device clearance preceded attempt 2, this measurement. NOT ANSWERED, counted wrong: exactly the 15 pre-registered long items exceed the author's 32,768-token limit (HTTP 400), at positions in the pre-registered long-item set; never truncated. Answered: 1,485 of 1,500 items sent, HTTP 200; all observed refusal ids and positions match the preregistration, no unexpected failures. 10 s RULE: 0 decisions over 10 s, so no decision is scored wrong under the 10 s rule in the v1.7 share view (the frozen v1.6 and v1.7-off views apply no time limit); maximum 0.294317 s (median 0.092944 s, p95 0.161959 s) over the 1,500 requests. DETERMINISM: 50 positions repeated, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0, on the server's rounded outputs. CONTEXT: author limit 32,768 tokens; no truncation. Over the cost cap under the earlier list-price basis; inside both caps from v1.8.0 (GPU-time cost basis).",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9090909090909091,
       "cc": 69.1088888888889,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 15.660000000000029
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 69.1088888888889
      },
      "calibration_gen": {
       "accuracy": 0.6534653465346535,
       "cc": 45.135582140332275,
       "cc_by_type": {
        "choice": 49.09862142099682,
        "noul": 28.000000000000004,
        "score": 58.308125
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 45.135582140332275
      },
      "composite_gen": {
       "accuracy": 0.9108910891089109,
       "cc": 87.7554431010122,
       "cc_by_type": {
        "choice": 96.7479674796748,
        "noul": 92.59259259259258,
        "score": 73.9257692307692
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 87.7554431010122
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 83.34037037037037,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 90.02111111111111
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 83.34037037037037
      },
      "jaggedness": {
       "accuracy": 0.7111111111111111,
       "cc": 49.24603884811041,
       "cc_by_type": {
        "choice": 66.79499518768047,
        "noul": 13.513513513513509,
        "score": 67.42960784313726
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 49.24603884811041
      },
      "llm_router": {
       "accuracy": 0.9329073482428115,
       "cc": 84.61928322280761,
       "cc_by_type": {
        "choice": 96.03768600862904,
        "noul": 95.8762886597938,
        "score": 61.943875000000006
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 84.61928322280761
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 33.14086892488955,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 0,
        "score": 48.90714285714287
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 33.14086892488955
      },
      "multilingual": {
       "accuracy": 0.7672727272727272,
       "cc": 59.326481019702726,
       "cc_by_type": {
        "choice": 79.45875448320835,
        "noul": 36.111111111111114,
        "score": 62.40957746478871
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 59.326481019702726
      },
      "negation_pair_gen": {
       "accuracy": 0.8837209302325582,
       "cc": 74.49422222222223,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 73.33333333333334,
        "score": 56.816
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 74.49422222222223
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.84375,
       "cc": 79.16758373758374,
       "cc_by_type": {
        "choice": 81.98198198198199,
        "noul": 100,
        "score": 55.52076923076924
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 79.16758373758374
      },
      "usecase": {
       "accuracy": 0.7181818181818181,
       "cc": 54.12956209150327,
       "cc_by_type": {
        "choice": 76.7156862745098,
        "noul": 25,
        "score": 60.673
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 54.12956209150327
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 45.03722832722832,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": 23.20692307692307
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 45.03722832722832
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 42.36644444444443,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": -6.234000000000028
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 42.36644444444443
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 80.36392458637937,
       "cc_by_type": {
        "choice": 60.176991150442475,
        "noul": 100,
        "score": 80.91478260869566
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 80.36392458637937
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.478482309124765,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 99.95500000000001
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.478482309124765
      },
      "en": {
       "accuracy": 0.8140747176368376,
       "cc": 68.59329552513215,
       "cc_by_type": {
        "choice": 81.98703945521775,
        "noul": 60.431654676258994,
        "score": 63.36119244391971
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 68.59329552513215
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.27265137702716,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 29.60588235294116
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.27265137702716
      },
      "fi": {
       "accuracy": 0.75,
       "cc": 66.14881602914389,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 33.33333333333333,
        "score": 97.89999999999998
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 66.14881602914389
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 95.69543859649123,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 87.08631578947367
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 95.69543859649123
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 48.9343156424581,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 25,
        "score": 55.322500000000005
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 48.9343156424581
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 87.97631778198418,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 77.95857142857145
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 87.97631778198418
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 38.191514782455215,
       "cc_by_type": {
        "choice": 71.40587768069896,
        "noul": 19.999999999999996,
        "score": 23.16866666666668
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 38.191514782455215
      },
      "ko": {
       "accuracy": 0.7,
       "cc": 24.918809523809518,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 74.75642857142856
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 24.918809523809518
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.8872222222222,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 97.32833333333333
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.8872222222222
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 0.7546580086580121,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": 28.237999999999996
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0.7546580086580121
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 50,
       "cc_by_type": {
        "choice": 100,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 50
      },
      "pl": {
       "accuracy": 0.7692307692307693,
       "cc": 57.96418817886866,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 19.999999999999996,
        "score": 61.509999999999984
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 57.96418817886866
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 80.66579447602031,
       "cc_by_type": {
        "choice": 78.78609310548026,
        "noul": 100,
        "score": 63.21129032258066
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 80.66579447602031
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 40.81872340425533,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 13.520000000000021
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 40.81872340425533
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 64.54112318840579,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": 15.362500000000011
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 64.54112318840579
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 86.98397849462366,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 93.21
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 86.98397849462366
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 45.65444444444446,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 3.630000000000022
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 45.65444444444446
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.856,
       "cc": 81.40033295700262,
       "cc_by_type": {
        "choice": 81.40033295700262
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 81.40033295700262
      },
      "noul": {
       "accuracy": 0.7813333333333333,
       "cc": 56.266666666666666,
       "cc_by_type": {
        "noul": 56.266666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 56.266666666666666
      },
      "score": {
       "accuracy": 0.728,
       "cc": 62.235198555956686,
       "cc_by_type": {
        "score": 62.235198555956686
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 62.235198555956686
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 17.159411764705876,
       "cc_by_type": {
        "choice": 84,
        "noul": -50,
        "score": 17.478235294117628
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 17.159411764705876
      },
      "code_linting": {
       "accuracy": 0.875,
       "cc": 76.62366539257845,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 100,
        "score": 67.13807692307694
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 76.62366539257845
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 41.596558265582665,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 7.3100000000000165
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 41.596558265582665
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -26.405562865497085,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": -100,
        "score": -9.041250000000023
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 40.86371980676329,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 84.91
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 40.86371980676329
      },
      "feature_extraction": {
       "accuracy": 0.7894736842105263,
       "cc": 45.7403668763103,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": 29.045000000000076
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 45.7403668763103
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 46.06900883430294,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": 14.28571428571428,
        "score": 53.333076923076916
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 46.06900883430294
      },
      "gaming": {
       "accuracy": 0.7727272727272727,
       "cc": 51.76553451497248,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 0,
        "score": 65.74956521739131
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 51.76553451497248
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 13.172037037037038,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 62.016111111111115
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 13.172037037037038
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 78.19744576384834,
       "cc_by_type": {
        "choice": 87.15203426124197,
        "noul": 66.66666666666667,
        "score": 80.77363636363637
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 78.19744576384834
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 60.99109427609428,
       "cc_by_type": {
        "choice": 100,
        "noul": 55.55555555555556,
        "score": 27.417727272727287
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 60.99109427609428
      },
      "legal_compliance": {
       "accuracy": 0.3,
       "cc": -0.41153673835126386,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 33.33333333333333,
        "score": -37.793750000000024
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.7407407407407407,
       "cc": 55.56472222222222,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 46.69416666666666
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 55.56472222222222
      },
      "model_routing": {
       "accuracy": 0.9078341013824884,
       "cc": 80.70044180645743,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 96.96969696969697,
        "score": 49.806953125
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.70044180645743
      },
      "moderation_trust_safety": {
       "accuracy": 0.7777777777777778,
       "cc": 18.187698995145087,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": -100,
        "score": 66.07941176470588
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 18.187698995145087
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 34.23259683832108,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": -33.333333333333336,
        "score": 69.45730769230771
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 34.23259683832108
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 30.910332270839376,
       "cc_by_type": {
        "choice": 70.44334975369459,
        "noul": 0,
        "score": 22.287647058823534
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 30.910332270839376
      },
      "scientific_discovery": {
       "accuracy": 0.6086956521739131,
       "cc": 30.59180193236715,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 68.587
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 30.59180193236715
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.88903508771931,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 96.6671052631579
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.88903508771931
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      87.13496634601603,
      92.25078661376436
     ],
     "I": [
      62.7370903412337,
      71.4169057679515
     ],
     "capability": [
      75.79878347672776,
      81.10866180633397
     ],
     "composite": null,
     "seed": 82
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.31679956749362,
     "capability": 79.55177804251245,
     "intelligence": 67.78675651753129
    },
    "per_type": {
     "choice": {
      "accuracy": 0.856,
      "calibration": 91.631737704918,
      "cc": 81.40033295700262,
      "cc_by_type": {
       "choice": 81.40033295700262
      },
      "cc_tiered": 81.67444257511333,
      "cc_tiered_open": 83.61812666623719,
      "cc_tiered_sealed": 79.73075848398948,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 81.40033295700262
     },
     "noul": {
      "accuracy": 0.7813333333333333,
      "calibration": 90.57538873994639,
      "cc": 56.266666666666666,
      "cc_by_type": {
       "noul": 56.266666666666666
      },
      "cc_tiered": 59.57893420847392,
      "cc_tiered_open": 69.94824016563147,
      "cc_tiered_sealed": 49.20962825131636,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 56.266666666666666
     },
     "score": {
      "accuracy": 0.728,
      "calibration": 91.7432722576165,
      "cc": 62.235198555956686,
      "cc_by_type": {
       "score": 62.235198555956686
      },
      "cc_tiered": 62.106892769006606,
      "cc_tiered_open": 64.4223575301033,
      "cc_tiered_sealed": 59.791428007909914,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 62.235198555956686
     }
    },
    "raw_on_A": {
     "calibration": 88.88233246209342,
     "capability": 78.24991223853809,
     "intelligence": 67.61749201498274
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 68.19245786851681,
    "Emphasis on Accuracy 60:20:20": 68.02959597388424,
    "Emphasis on Cost 20:20:60": 62.69429962113339,
    "Emphasis on Speed 20:60:20": 74.94433405764518,
    "Intelligence only": 67.78675651753129
   }
  },
  {
   "adapter_id": "images/verdict-small/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Manavarya09 (Manav)",
   "axes": {
    "calibration": 72.92000413192591,
    "cost": 100,
    "intelligence": 2.3417095945172277,
    "speed": 88.74744771425449
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13790718750000003,
      "ece_score": 72.4185625,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 72.4185625,
      "score_before_time_limit": 72.4185625,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.5546666666666666,
      "brier": 0.28117829917333326,
      "ece": 0.18824453333333335,
      "n": 375,
      "score": 62.35109333333333,
      "score_before_time_limit": 62.35109333333333,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.984,
      "interval90_width": 0.9684444444444443,
      "n": 375,
      "nrps": 0.18136406875111122,
      "score": 83.99035656244443,
      "score_before_time_limit": 83.99035656244443,
      "time_limit_failed_share": 0,
      "top_ece": 0.06941440000000007
     }
    },
    "score": 72.92000413192591
   },
   "capability": 37.63085686322157,
   "class": "jev-rebuild",
   "composite_ci95": {
    "A": [
     0.0003475070464105525,
     0.12852579345393955
    ],
    "B": [
     0.00021949435923644138,
     0.08621537426131828
    ],
    "C": [
     0.0002413243377851058,
     0.08925402323190244
    ]
   },
   "cost": {
    "basis": "v1.5.4 published cost per 1,000 decisions carried (pricing rules unchanged; v1.6 item lengths differ): documented hosted-model estimate; no exact base-model floor applies",
    "common_basis": null,
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.000866152709359606
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "verdict-small (Manavarya09, multilingual-e5-small 118M)",
   "endpoint_condition": "evaluator-owned CPU container, offline read-only, pinned cores",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "cpu",
   "intelligence": {
    "I_open": 4.005009650747796,
    "I_sealed": 0.6784095382866591,
    "base": 2.3417095945172277,
    "excess": 0.7532124700228935,
    "gap": 3.3266001124611373,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 12.01502895224339,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": -8.972972972972968,
       "hard": 13.180245360176167,
       "judge": 9.03098220171391,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -28.57142857142857,
       "hard": -39.13043478260869,
       "judge": -40,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 6.291818181818199,
       "hard": -6.231481481481493,
       "judge": 7.341694915254216,
       "standard": -11.9677777777778
      }
     },
     "sealed|choice": {
      "cc": 2.035228614859977,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": -7.75256817677922,
       "hard": -4.355203619909522,
       "judge": 6.890130353817524,
       "standard": 12.427638871782255
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -31.999999999999996,
       "hard": -45.63106796116505,
       "judge": -47.36842105263158,
       "standard": -46.47887323943662
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": -0.7665248226950139,
       "hard": -6.760098039215645,
       "judge": 6.645688622754465,
       "standard": -2.297512690355341
      }
     }
    }
   },
   "jevbench_score": 0.018990012754194416,
   "key": "verdict-small",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0 (verdictml code and the Manav2op/verdict-small checkpoint over intfloat/multilingual-e5-small)",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6227544910179641,
    "decisive_rate": 0.44533333333333336,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 72.92000413192591,
     "cost": 100,
     "intelligence": 2.3417095945172277,
     "speed": 89.48731594094681
    },
    "board_release": "v1.6.43",
    "capability": 37.63085686322157,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 119,
    "capability_rank": 147,
    "composite_ci95": {
     "A": [
      0.00034752623458969486,
      0.12857270817562724
     ],
     "B": [
      0.00021950048318305755,
      0.08623222919712645
     ],
     "C": [
      0.0002413376629095103,
      0.08928660289974111
     ]
    },
    "jevbench_score": 0.01899384266854915,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 187,
    "rank": 231,
    "ranks": {
     "A": 231,
     "B": 231,
     "C": 231,
     "capability": 211
    },
    "scores": {
     "A": 0.01899384266854915,
     "B": 0.012337048142708213,
     "C": 0.013190168519825799
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.24820413775742053,
     "p50_s_raw": 0.04910206887871027,
     "p95_s_adjusted": 0.4533769000321627,
     "p95_s_raw": 0.15168845001608133
    },
    "views": {
     "Balanced 33:33:33": 0.014681237741460948,
     "Emphasis on Accuracy 60:20:20": 0.008421478349576572,
     "Emphasis on Cost 20:20:60": 0.02342353351655187,
     "Emphasis on Speed 20:60:20": 0.02330657899907492,
     "Intelligence only": 0.005136403075864165
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 272,
   "ranked": true,
   "ranks": {
    "A": 272,
    "B": 272,
    "C": 272,
    "capability": 252
   },
   "repo": "https://github.com/Manavarya09/verdict",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.018990012754194416,
    "B": 0.01233575538253284,
    "C": 0.013187508857079457
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.28037555478513243,
    "p50_s_raw": 0.0651877773925662,
    "p95_s_adjusted": 0.47589931823313236,
    "p95_s_raw": 0.1629496591165662
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "intfloat/multilingual-e5-small (118M multilingual bi-encoder) fine-tuned on a typed-decision mix; each option is scored against the rendered state by cosine similarity, with temperature scaling and a conformal abstain set on top. One encoder pass per option, no tokens generated.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      67.28225057545556,
      76.20018850659073
     ],
     "I": [
      0.6053648936331004,
      4.532806367679299
     ],
     "capability": [
      34.685562893137785,
      39.80647930109692
     ],
     "composite": {
      "A": [
       0.0003475070464105525,
       0.12852579345393955
      ],
      "B": [
       0.00021949435923644138,
       0.08621537426131828
      ],
      "C": [
       0.0002413243377851058,
       0.08925402323190244
      ]
     },
     "seed": 108
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 72.92000413192591,
     "capability": 37.63085686322157,
     "intelligence": 2.3417095945172277
    },
    "per_type": {
     "choice": {
      "accuracy": 0.23733333333333334,
      "calibration": 72.4185625,
      "cc": 2.4658260943701484,
      "cc_by_type": {
       "choice": 2.4658260943701484
      },
      "cc_tiered": 7.0251287835516845,
      "cc_tiered_open": 12.01502895224339,
      "cc_tiered_sealed": 2.035228614859977,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 2.4658260943701484
     },
     "noul": {
      "accuracy": 0.2773333333333333,
      "calibration": 62.35109333333333,
      "cc": -44.53333333333334,
      "cc_by_type": {
       "noul": -44.53333333333334
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.2986666666666667,
      "calibration": 83.99035656244443,
      "cc": -2.2494035087719633,
      "cc_by_type": {
       "score": -2.2494035087719633
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 74.70758393102564,
     "capability": 38.932253022553525,
     "intelligence": 3.156922114081418
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.014678186863820933,
    "Emphasis on Accuracy 60:20:20": 0.00842087594735804,
    "Emphasis on Cost 20:20:60": 0.023418873801355072,
    "Emphasis on Speed 20:60:20": 0.023292744564727425,
    "Intelligence only": 0.005136403075864165
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Victor Bennett (Hugging Face victorbennett; benchmarkheaven.com/submit ref E59F3C77)",
   "axes": {
    "calibration": 83.75263461013449,
    "cost": 76.72117336001418,
    "intelligence": 35.44491489257096,
    "speed": 90.21057479487555
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07788559948821713,
      "ece_score": 84.42288010235657,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 84.42288010235657,
      "score_before_time_limit": 84.42288010235657,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.34584450402144773,
      "brier": 0.16549543694752686,
      "ece": 0.10689071429896141,
      "n": 373,
      "score": 78.62185714020772,
      "score_before_time_limit": 78.62185714020772,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9516129032258065,
      "interval90_width": 0.603942652329749,
      "n": 372,
      "nrps": 0.10859985038392947,
      "score": 88.21316658783921,
      "score_before_time_limit": 88.21316658783921,
      "time_limit_failed_share": 0,
      "top_ece": 0.06356840892964317
     }
    },
    "score": 83.75263461013449
   },
   "capability": 59.59877475135272,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 902,188 input + 0 output tokens over 1,485 decisions = 607.534006734 tokens/decision. GPU-time cost USD 0.00596981520777629/1,000; old list-price cost USD 0.0182260202020202/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00596981520777629/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.018043760000000002
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.00596981520777629
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Verdin-4B (Victor Bennett, Qwen3.5-4B + merged LoRA r16 + pointer head)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-22",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Verda RTX 6000 Ada (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-01, one dedicated GPU instance, network none (proven), evaluator-controlled self-hosted deployment. The author's own Verdin engine and server, unmodified, at their declared GPU configuration: `python3 -m verdin.server --model <pinned snapshot> --host 127.0.0.1 --port <p> --backend cuda --dtype bf16 --name verdin-4b` (the author's CUDA bf16 path; fast gated-delta kernels, causal-conv1d, prefill chunk 8192, max sweep 16,384 tokens / 256 rows, server batching defaults), without the earlier CPU run's float32 and `--max-batch 1` (both forbidden by the gate and absent from the launched argv/env). Stack: Python 3.12 venv, torch 2.8.0+cu128 and the other pins of the author's own requirements-cuda.txt (a packaging change only: the author's pinned numpy needs Python 3.12, which the earlier pinned base image did not ship; weights and author code are unchanged and identical to the first use). The declared hardware of the author is one RTX 5090; the evaluator's GPU is an RTX 6000 Ada, the only deviation from the declared configuration. The author's renderer refuses a state over its own 65,536-token cap with HTTP 422 rather than truncating, so the draw's 15 longest items are honest refusals scored as wrong (the same 15 items as in the CPU row); the offline recount matched the pre-registered forecast exactly (1,485 answered, 15 refused, 902,188 input tokens). Speed is a dedicated-GPU figure, measured one request at a time; the standard self-hosted x2 + 0.15 s adjustment applies and raw p50/p95 are kept. RE-MEASURE ON THE SAME ITEMS: this GPU row REPLACES the earlier loaded-CPU row in full (scores AND times); the GPU times are not combined with the CPU scores. The same 1,500 items of draw v1.6-regular-20261012 (input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b), the same pinned weights. Pod: Verda 1RTX6000ADA.10V (NVIDIA RTX 6000 Ada Generation, 49,140 MiB), FIN-01, torn down after the run (22 Oct 2026). The scoring run had no network: container NetworkMode none, only the loopback interface, cut proven by three failed outbound probes before the first sealed byte; read-only root, all capabilities dropped. The earlier CPU measurement of this row was its first use of the weights on this draw; this is the second and last.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 48.98952596706263,
    "I_sealed": 25.859907489946018,
    "base": 37.42471672850432,
    "excess": 13.290091706760904,
    "gap": 23.129618477116615,
    "penalty": 0.947099082932391,
    "per_type_split": {
     "open|choice": {
      "cc": 69.12490790461862,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243244,
       "hard": 64.1396665618119,
       "judge": 80.22412656558998,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 37.862663906142174,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 56.52173913043479,
       "judge": -9.999999999999998,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 39.981006090427115,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 46.24734845438208,
       "hard": 42.544026794962654,
       "judge": 42.50635554607598,
       "standard": 27.933769315905277
      }
     },
     "sealed|choice": {
      "cc": 53.08980922463883,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 47.958829651902384,
       "judge": 49.75514201762979,
       "standard": 57.28749209027631
      }
     },
     "sealed|noul": {
      "cc": 0.8830767791138365,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 20.68965517241379,
       "hard": -3.6363636363636376,
       "judge": -19.999999999999996,
       "standard": 31.343283582089555
      }
     },
     "sealed|score": {
      "cc": 23.606836466085404,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 48.65018364828838,
       "hard": 16.741050238290057,
       "judge": 13.688946355597997,
       "standard": 39.69357049630571
      }
     }
    }
   },
   "jevbench_score": 31.275579231606862,
   "key": "verdin-4b",
   "last_measured_on": "2026-10-22",
   "licence": "Weights CC-BY-NC-4.0 (non-commercial; the repo's NOTICE and the model card state that part of the training mix - BeaverTails, the ANLI translations, RAGTruth's MS MARCO and Yelp-derived contexts - is non-commercial). Base model Qwen/Qwen3.5-4B Apache-2.0 (LICENSE.base). The author's Verdin engine, shipped in the same repo, Apache-2.0. LICENCE FLAG (NC): CC-BY-NC-4.0 (non-commercial) weights; listing a measured score is evaluation use.",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "victorbennett/Verdin-4B@f35e9c184864a2fd6f286166783b0321d95f23b0",
   "noul_decisive": {
    "acc_among_decisive": 0.8647540983606558,
    "decisive": 244,
    "decisive_rate": 0.6541554959785523,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.57547250683973,
     "cost": 62.17924458049355,
     "intelligence": 35.50482714952485,
     "speed": 45.8869739530126
    },
    "board_release": "v1.6.43",
    "capability": 59.540149828182294,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      17.04093820678688,
      27.484134104815304
     ],
     "B": [
      15.411632616514813,
      25.68836113806341
     ],
     "C": [
      11.833984865824224,
      19.08620423945507
     ]
    },
    "jevbench_score": 21.77748190312391,
    "measured_in": "v1.6.27",
    "method": "v1.6",
    "open_board_rank": 84,
    "rank": 117,
    "ranks": {
     "A": 117,
     "B": 130,
     "C": 135,
     "capability": 136
    },
    "scores": {
     "A": 21.77748190312391,
     "B": 20.00039057273753,
     "C": 15.123251321613825
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 35.47302223105216,
     "p50_s_raw": 17.661511115526082,
     "p95_s_adjusted": 72.67824198754965,
     "p95_s_raw": 36.264120993774824
    },
    "views": {
     "Balanced 33:33:33": 19.292352873017215,
     "Emphasis on Accuracy 60:20:20": 17.352655644184253,
     "Emphasis on Cost 20:20:60": 21.62260137257583,
     "Emphasis on Speed 20:60:20": 19.370063474123214,
     "Intelligence only": 15.078600269727296
    }
   },
   "rank": 161,
   "ranked": true,
   "ranks": {
    "A": 161,
    "B": 164,
    "C": 163,
    "capability": 167
   },
   "remeasure": {
    "disclosure": "Re-measure on the same items: this GPU row replaces the earlier loaded-CPU row of the same weights in full (scores and times) and is the second and last use of these weights on draw v1.6-regular-20261012.",
    "draw": "v1.6-regular-20261012",
    "kind": "re-measure on the same items",
    "replaces": {
     "board_release": "v1.6.43",
     "endpoint_kind": "cpu",
     "in_full": "scores and times"
    }
   },
   "remeasured_in": "v1.7.0",
   "repo": "https://huggingface.co/victorbennett/Verdin-4B",
   "scores": {
    "A": 31.275579231606862,
    "B": 27.16858841504966,
    "C": 21.719152244171436
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2660857129998476,
    "p50_s_raw": 0.0580428564999238,
    "p95_s_adjusted": 0.35803127409999425,
    "p95_s_raw": 0.10401563704999713
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8181818181818182,
       "cc": 61.523990722530975,
       "cc_by_type": {
        "choice": 75,
        "noul": 100,
        "score": 9.571972167592913
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 61.523990722530975
      },
      "calibration_gen": {
       "accuracy": 0.42574257425742573,
       "cc": 14.794406109571346,
       "cc_by_type": {
        "choice": 26.82926829268293,
        "noul": 4.0000000000000036,
        "score": 13.553950036031104
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 14.794406109571346
      },
      "composite_gen": {
       "accuracy": 0.504950495049505,
       "cc": 7.520948912108281,
       "cc_by_type": {
        "choice": 57.72357723577236,
        "noul": -48.14814814814815,
        "score": 12.987417648700639
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 7.520948912108281
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 73.41803255349417,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 60.25409766048252
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 73.41803255349417
      },
      "jaggedness": {
       "accuracy": 0.46111111111111114,
       "cc": 9.326554298072596,
       "cc_by_type": {
        "choice": 37.92107795957652,
        "noul": -40.54054054054054,
        "score": 30.59912547518181
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 9.326554298072596
      },
      "llm_router": {
       "accuracy": 0.7507987220447284,
       "cc": 59.92658024610777,
       "cc_by_type": {
        "choice": 69.09395086730652,
        "noul": 58.762886597938135,
        "score": 51.922903273078646
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 59.92658024610777
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4444444444444444,
       "cc": 0.44581408543078044,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -25,
        "score": 0.5642463800036857
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0.44581408543078044
      },
      "multilingual": {
       "accuracy": 0.6254545454545455,
       "cc": 34.88131457686409,
       "cc_by_type": {
        "choice": 58.9175089664167,
        "noul": 13.888888888888884,
        "score": 31.8375458752867
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 34.88131457686409
      },
      "negation_pair_gen": {
       "accuracy": 0.47674418604651164,
       "cc": 6.166223129941823,
       "cc_by_type": {
        "choice": 46.666666666666664,
        "noul": -26.66666666666667,
        "score": -1.5013306101745227
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 6.166223129941823
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.78125,
       "cc": 66.91900260860074,
       "cc_by_type": {
        "choice": 69.96996996996997,
        "noul": 100,
        "score": 30.787037855832256
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 66.91900260860074
      },
      "usecase": {
       "accuracy": 0.509090909090909,
       "cc": 24.52631268483532,
       "cc_by_type": {
        "choice": 54.65686274509807,
        "noul": -8.333333333333337,
        "score": 27.25540864274124
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 24.52631268483532
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": 5.110110788373182,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50,
        "score": -58.47919144440426
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 5.110110788373182
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.5384615384615384,
       "cc": 13.276437832483587,
       "cc_by_type": {
        "choice": 58.52534562211981,
        "noul": -33.333333333333336,
        "score": 14.637301208664288
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 13.276437832483587
      },
      "de": {
       "accuracy": 0.6875,
       "cc": 53.55726914887933,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 75,
        "score": 33.45941806610703
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 53.55726914887933
      },
      "el": {
       "accuracy": 0.4,
       "cc": 21.344756984735273,
       "cc_by_type": {
        "choice": 32.960893854748605,
        "noul": -66.66666666666667,
        "score": 97.74004376612389
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 21.344756984735273
      },
      "en": {
       "accuracy": 0.5881841876629018,
       "cc": 31.086408945268783,
       "cc_by_type": {
        "choice": 56.50529039186729,
        "noul": 7.913669064748197,
        "score": 28.840267379190866
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 31.086408945268783
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 56.4252784262924,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": -11.148308277403407
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 56.4252784262924
      },
      "fi": {
       "accuracy": 0.25,
       "cc": 5.744506156424248,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": -33.333333333333336,
        "score": 48.927507540311
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 5.744506156424248
      },
      "fr": {
       "accuracy": 0.5714285714285714,
       "cc": 34.65178850066019,
       "cc_by_type": {
        "choice": 36.47058823529412,
        "noul": 0,
        "score": 67.48477726668645
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 34.65178850066019
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 34.34458127099561,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 36.55329688561254
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 34.34458127099561
      },
      "id": {
       "accuracy": 0.5,
       "cc": -22.222222222222225,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 68.88724101042284,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 20.691341113887418
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 68.88724101042284
      },
      "ja": {
       "accuracy": 0.72,
       "cc": 51.95211351224185,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 80,
        "score": 33.04458517532763
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 51.95211351224185
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 5.151819689710763,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 15.455459069132294
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 5.151819689710763
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 92.68822847084569,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 78.0646854125371
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 92.68822847084569
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 23.297635944016566,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 55.17429311343498
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 23.297635944016566
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 40.04671144489828,
       "cc_by_type": {
        "choice": 77.14769360981802,
        "noul": 19.999999999999996,
        "score": 22.99244072487682
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 40.04671144489828
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 51.04746948866693,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 35.570222255040264
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 51.04746948866693
      },
      "sv": {
       "accuracy": 0.36363636363636365,
       "cc": 19.073419744761807,
       "cc_by_type": {
        "choice": 23.404255319148945,
        "noul": -19.999999999999996,
        "score": 53.81600391513648
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 19.073419744761807
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 25.79911330832609,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 20.87560079454349
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 25.79911330832609
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 66.65097394291317,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 32.21098634486855
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 66.65097394291317
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 14.386174718477827,
       "cc_by_type": {
        "choice": 74.57627118644068,
        "noul": -33.333333333333336,
        "score": 1.9155863023261444
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 14.386174718477827
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.668,
       "cc": 57.117434317533835,
       "cc_by_type": {
        "choice": 57.117434317533835
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 57.117434317533835
      },
      "noul": {
       "accuracy": 0.5626666666666666,
       "cc": 12.53333333333333,
       "cc_by_type": {
        "noul": 12.53333333333333
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 12.53333333333333
      },
      "score": {
       "accuracy": 0.5013333333333333,
       "cc": 29.091210398518484,
       "cc_by_type": {
        "score": 29.091210398518484
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 29.091210398518484
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.36363636363636365,
       "cc": -7.421017262210398,
       "cc_by_type": {
        "choice": 52,
        "noul": -75,
        "score": 0.7369482133688043
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 0
      },
      "code_linting": {
       "accuracy": 0.375,
       "cc": 1.696566639321733,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": -11.111111111111116,
        "score": -9.265027480240452
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 1.696566639321733
      },
      "customer_support": {
       "accuracy": 0.7857142857142857,
       "cc": 53.72603827573835,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 43.698440030467054
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 53.72603827573835
      },
      "demand_forecasting": {
       "accuracy": 0.29411764705882354,
       "cc": -1.3767897429531726,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": -33.333333333333336,
        "score": 22.770215566462127
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.47058823529411764,
       "cc": -8.940606953354225,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -100,
        "score": 31.14919363269094
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 0
      },
      "feature_extraction": {
       "accuracy": 0.42105263157894735,
       "cc": -7.0516993000313795,
       "cc_by_type": {
        "choice": 11.949685534591195,
        "noul": -33.333333333333336,
        "score": 0.22854989864800057
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.5,
       "cc": 26.75005211407454,
       "cc_by_type": {
        "choice": 70.58823529411765,
        "noul": -14.28571428571429,
        "score": 23.947635333820262
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 26.75005211407454
      },
      "gaming": {
       "accuracy": 0.36363636363636365,
       "cc": -8.642668549110079,
       "cc_by_type": {
        "choice": 37.2822299651568,
        "noul": -100,
        "score": 36.78976438751296
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 20.188784788594514,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": 19.999999999999996,
        "score": 23.89968769911688
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 20.188784788594514
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 41.54483291107334,
       "cc_by_type": {
        "choice": 35.76017130620985,
        "noul": 66.66666666666667,
        "score": 22.207660760343508
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 41.54483291107334
      },
      "lead_generation": {
       "accuracy": 0.6,
       "cc": 37.187525164753104,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 33.33333333333333,
        "score": 1.4850561144143515
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 37.187525164753104
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": -5.059825983930627,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": -100,
        "score": 17.07858656433716
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.5555555555555556,
       "cc": 27.97647267533547,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -19.999999999999996,
        "score": 26.962047584547655
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 27.97647267533547
      },
      "model_routing": {
       "accuracy": 0.7788018433179723,
       "cc": 61.48579662948972,
       "cc_by_type": {
        "choice": 81.2987012987013,
        "noul": 63.63636363636365,
        "score": 39.52232495340419
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 61.48579662948972
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 5.86650020310321,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 52.14844494712153
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 5.86650020310321
      },
      "recruiting": {
       "accuracy": 0.6111111111111112,
       "cc": 62.59236833041763,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 100,
        "score": 21.203288835264043
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 62.59236833041763
      },
      "risk_assessment": {
       "accuracy": 0.3888888888888889,
       "cc": 1.030281616673367,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": -50,
        "score": 12.204145342630934
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 1.030281616673367
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 15.64194265939215,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": 0,
        "score": 19.389596094118478
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 15.64194265939215
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 73.80007572716669,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 71.40022718150007
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 73.80007572716669
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.79608109938583,
      84.57493113282798
     ],
     "I": [
      32.107645595824906,
      39.19687931973971
     ],
     "capability": [
      55.812920523721445,
      60.99580491591795
     ],
     "composite": null,
     "seed": 82
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.75263461013449,
     "capability": 59.59877475135272,
     "intelligence": 35.44491489257096
    },
    "per_type": {
     "choice": {
      "accuracy": 0.668,
      "calibration": 84.42288010235657,
      "cc": 57.117434317533835,
      "cc_by_type": {
       "choice": 57.117434317533835
      },
      "cc_tiered": 61.107358564628726,
      "cc_tiered_open": 69.12490790461862,
      "cc_tiered_sealed": 53.08980922463883,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 57.117434317533835
     },
     "noul": {
      "accuracy": 0.5626666666666666,
      "calibration": 78.62185714020772,
      "cc": 12.53333333333333,
      "cc_by_type": {
       "noul": 12.53333333333333
      },
      "cc_tiered": 19.372870342628005,
      "cc_tiered_open": 37.862663906142174,
      "cc_tiered_sealed": 0.8830767791138365,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 12.53333333333333
     },
     "score": {
      "accuracy": 0.5013333333333333,
      "calibration": 88.21316658783921,
      "cc": 29.091210398518484,
      "cc_by_type": {
       "score": 29.091210398518484
      },
      "cc_tiered": 31.79392127825626,
      "cc_tiered_open": 39.981006090427115,
      "cc_tiered_sealed": 23.606836466085404,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 29.091210398518484
     }
    },
    "raw_on_A": {
     "calibration": 74.90958669726717,
     "capability": 56.9855068754991,
     "intelligence": 39.061427053731016
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 28.80847779018964,
    "Emphasis on Accuracy 60:20:20": 23.10349119050496,
    "Emphasis on Cost 20:20:60": 32.049294887764674,
    "Emphasis on Speed 20:60:20": 33.72615741196566,
    "Intelligence only": 17.812373989297615
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "taoszhang (Hugging Face)",
   "axes": {
    "calibration": 94.3167842335138,
    "cost": 61.42265544156546,
    "intelligence": 78.78162736656175,
    "speed": 84.80563168345245
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03870041163456402,
      "ece_score": 92.2599176730872,
      "mean_tvd": null,
      "n": 746,
      "n_ece_hard": 350,
      "n_tvd": 0,
      "score": 92.2599176730872,
      "score_before_time_limit": 92.2599176730872,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.09625668449197866,
      "brier": 0.03667987144830499,
      "ece": 0.029673749008404773,
      "n": 374,
      "score": 94.06525019831905,
      "score_before_time_limit": 94.06525019831905,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.984,
      "interval90_width": 0.08977777777777775,
      "n": 375,
      "nrps": 0.02160277603233321,
      "score": 96.62518482913518,
      "score_before_time_limit": 96.62518482913518,
      "time_limit_failed_share": 0,
      "top_ece": 0.022946763692481702
     }
    },
    "score": 94.3167842335138
   },
   "capability": 86.54920580003778,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11763 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 559,606 input + 0 output tokens over 1,489 decisions = 375.826729349 tokens/decision. GPU-time cost USD 0.0193157459751563/1,000; old list-price cost USD 0.157847226326394/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0193157459751563/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1489,
     "rule": "common",
     "usd_per_1000_all_items": 0.15740227999999998
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.019315745975156336
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "disclosure": "VisionJEV 1.0.0 is a 27B model built on Qwen/Qwen3.8-27B, with a decision readout over the last-token hidden state. The repository NOTICE attributes the prompt, answer conversion and noncausal attention hook to autojev/pplx-decider. TEXT ONLY: the vision-capable model was measured without images. SERVING: the author's own shipped model and documented HTTP server, unmodified, BF16, SDPA, noncausal full attention, last-token pooling, temperature 0.986316004609301, batch size 1, no thinking, one forward pass, no truncation, quantisation, rereading or retries. Contributor code ran only on a disposable rented pod with network cut. ITEMS: all 1,500 items were sent; 5 exceeded the native 8,192-token input limit and were refused with HTTP 422, scored as misses and kept in the denominator; these carry no usage and are unbilled. 1,495 items were answered. TOKENS: output tokens reported as 0; native input usage is the observed length of input ids at the native prepare call, 562,151 input tokens on answered requests. TRAINING: training data were not inspected; overlap with public or sealed JevBench items is unchecked. DETERMINISM: 49 of 50 replayed requests were bit-identical, with 0 argmax flips (recorded maximum probability difference 0.0). WARM-UP: a public synthetic warm-up preceded the timed measurements; the first cold smoke request took about 42 seconds. LATENCY: measured loopback p50 0.152 s, p95 0.290 s, maximum 9.297 s. The share view applies the common self-hosted latency assumption, twice measured p50 plus 0.15 s, an assumption rather than a hosted-endpoint measurement. HARDWARE: one A100 80 GB GPU; weights alone occupy 48.6 GiB and smoke peak allocated memory was 50.2 GiB. LICENCE: Apache-2.0 with NOTICE preserved. The 27 Oct 2026 draw is not yet equated to the reference scale (offset 0). This draw has no subject-topic labels yet, so the row has no topic cells; use-case and language cells are measured on its own draw.",
   "display": "VisionJEV 1.0.0",
   "draw": {
    "drawn_on": "2026-10-27",
    "g_med": 2.573387642438244,
    "id": "v1.6-regular-20261027",
    "label": "Third regular draw",
    "measured_on": "2026-10-29",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.21",
    "results_sha256": "5bb4e9cdc8f26059f0802ff17ba138ff8db9642b9a92ecf714f9d3804d040686",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Self-hosted on one evaluator-rented Verda NVIDIA A100-SXM4-80GB in Finland (EU), using the author's own documented HTTP server on loopback; BF16 and SDPA. Measured 29 Oct 2026. Public warm-up preceded timed measurements. The model, source and input were read-only, network cut before scored input. GPU rental is separate from the estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 76.83635376036995,
    "I_sealed": 80.72690097275355,
    "base": 78.78162736656175,
    "excess": -6.4639348548218525,
    "gap": -3.890547212383609,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 84.8046790723018,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 83.01352626612143,
       "judge": 92.089650626236,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 77.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 50,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 68.62708752281769,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 73.56942570749214,
       "hard": 75.45575136614295,
       "judge": 69.95772006187286,
       "standard": 50.50264193524716
      }
     },
     "sealed|choice": {
      "cc": 86.78574812553104,
      "n": {
       "easy": 73,
       "hard": 286,
       "judge": 135,
       "standard": 106
      },
      "tiers": {
       "easy": 98.03471994759252,
       "hard": 78.13226283541096,
       "judge": 93.7877480586713,
       "standard": 87.9652328950301
      }
     },
     "sealed|noul": {
      "cc": 69.02255168501965,
      "n": {
       "easy": 87,
       "hard": 82,
       "judge": 45,
       "standard": 86
      },
      "tiers": {
       "easy": 95.40229885057472,
       "hard": 46.34146341463414,
       "judge": 82.22222222222221,
       "standard": 81.3953488372093
      }
     },
     "sealed|score": {
      "cc": 86.37240310770999,
      "n": {
       "easy": 68,
       "hard": 112,
       "judge": 56,
       "standard": 64
      },
      "tiers": {
       "easy": 96.89627448303031,
       "hard": 74.50853214246858,
       "judge": 93.51795670585594,
       "standard": 94.11987895331369
      }
     }
    }
   },
   "jevbench_score": 77.86920419622817,
   "key": "visionjev-1.0.0-27b",
   "last_measured_on": "2026-10-29",
   "licence": "Apache-2.0 + NOTICE",
   "listing": "unranked",
   "measured_in": "v1.7.21",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": {
    "base": "Qwen/Qwen3.8-27B",
    "model": "taoszhang/VisionJEV",
    "revision": "cd6353aaea57561bd8c9beee69311f104dd77528",
    "served_model": "VisionJEV 1.0.0"
   },
   "new_in": "v1.7.21",
   "not_ranked_because": "Measured on the 27 Oct 2026 draw, which is not yet equated to the reference scale; stronger anchor runs for that draw are being measured. Shown unranked until then.",
   "noul_decisive": {
    "acc_among_decisive": 0.9792899408284024,
    "decisive": 338,
    "decisive_rate": 0.9037433155080213,
    "n": 375,
    "supported": true,
    "valid": 374
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "provider": "self-hosted",
   "rank": null,
   "ranked": false,
   "ranks": {
    "A": null,
    "B": null,
    "C": null,
    "capability": null
   },
   "repo": "https://huggingface.co/taoszhang/VisionJEV",
   "scores": {
    "A": 77.86920419622817,
    "B": 78.04999412266767,
    "C": 77.86920419622817
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.4530805039998086,
    "p50_s_raw": 0.15154025199990429,
    "p95_s_adjusted": 0.7298967951001715,
    "p95_s_raw": 0.2899483975500857
   },
   "status": {
    "answered_ok": 1495,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 1,
        "noul": 3
       },
       "score": 100
      },
      "calibration_gen": {
       "accuracy": 0.7857142857142857,
       "cc": 69.5336643420706,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 100,
        "score": 60.54904497426372
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 5,
        "noul": 2,
        "score": 7
       },
       "score": 69.5336643420706
      },
      "composite_gen": {
       "accuracy": 0.8947368421052632,
       "cc": 89.51051335329407,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 68.53154005988219
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 9
       },
       "score": 89.51051335329407
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.99966623036187,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.99899869108563
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 99.99966623036187
      },
      "jaggedness": {
       "accuracy": 0.7272727272727273,
       "cc": 35.589767287461875,
       "cc_by_type": {
        "choice": 62.96296296296297,
        "noul": -33.333333333333336,
        "score": 77.139672232756
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 8
       },
       "score": 35.589767287461875
      },
      "llm_router": {
       "accuracy": 0.9111111111111111,
       "cc": 85.16582524388743,
       "cc_by_type": {
        "choice": 92.06815511163339,
        "noul": 100,
        "score": 63.42932062002888
       },
       "low_n": false,
       "n": 90,
       "n_by_type": {
        "choice": 45,
        "noul": 23,
        "score": 22
       },
       "score": 85.16582524388743
      },
      "long_state": {
       "accuracy": 0,
       "cc": -64.51612903225806,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100
       },
       "low_n": true,
       "n": 5,
       "n_by_type": {
        "choice": 4,
        "noul": 1
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 2,
       "n_by_type": {
        "choice": 1,
        "noul": 1
       },
       "score": 100
      },
      "multilingual": {
       "accuracy": 0.9011943539630836,
       "cc": 83.62171862685402,
       "cc_by_type": {
        "choice": 85.77279586117699,
        "noul": 77.04918032786885,
        "score": 88.0431796915162
       },
       "low_n": false,
       "n": 921,
       "n_by_type": {
        "choice": 430,
        "noul": 244,
        "score": 247
       },
       "score": 83.62171862685402
      },
      "negation_pair_gen": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 12
       },
       "score": 100
      },
      "option_order": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 27
       },
       "score": 100
      },
      "perturb_pair": {
       "accuracy": 0.8571428571428571,
       "cc": 79.03221477448402,
       "cc_by_type": {
        "choice": 81.13207547169812,
        "noul": 80,
        "score": 75.96456885175392
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 14,
        "noul": 10,
        "score": 4
       },
       "score": 79.03221477448402
      },
      "usecase": {
       "accuracy": 0.88,
       "cc": 78.35764232538214,
       "cc_by_type": {
        "choice": 86.14150255288111,
        "noul": 63.07692307692307,
        "score": 85.85450134634223
       },
       "low_n": false,
       "n": 325,
       "n_by_type": {
        "choice": 188,
        "noul": 65,
        "score": 72
       },
       "score": 78.35764232538214
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.9047619047619048,
       "cc": 92.25385358093003,
       "cc_by_type": {
        "choice": 86.3013698630137,
        "noul": 100,
        "score": 90.46019087977639
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 5
       },
       "score": 92.25385358093003
      },
      "cs": {
       "accuracy": 0.8378378378378378,
       "cc": 74.1514552089692,
       "cc_by_type": {
        "choice": 67.91443850267379,
        "noul": 55.55555555555556,
        "score": 98.98437156867823
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 74.1514552089692
      },
      "da": {
       "accuracy": 0.8055555555555556,
       "cc": 67.4685276281638,
       "cc_by_type": {
        "choice": 77.63975155279502,
        "noul": 45.45454545454546,
        "score": 79.31128587715092
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 19,
        "noul": 11,
        "score": 6
       },
       "score": 67.4685276281638
      },
      "de": {
       "accuracy": 0.8958333333333334,
       "cc": 84.38106562927382,
       "cc_by_type": {
        "choice": 86.10753794700281,
        "noul": 75,
        "score": 92.03565894081865
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 30,
        "noul": 8,
        "score": 10
       },
       "score": 84.38106562927382
      },
      "el": {
       "accuracy": 0.9285714285714286,
       "cc": 84.9130496030355,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 94.7391488091065
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 10,
        "score": 6
       },
       "score": 84.9130496030355
      },
      "en": {
       "accuracy": 0.8846846846846846,
       "cc": 80.72499580624702,
       "cc_by_type": {
        "choice": 86.24904507257448,
        "noul": 73.77049180327869,
        "score": 82.1554505428879
       },
       "low_n": false,
       "n": 555,
       "n_by_type": {
        "choice": 311,
        "noul": 122,
        "score": 122
       },
       "score": 80.72499580624702
      },
      "es": {
       "accuracy": 0.918918918918919,
       "cc": 87.18835142790839,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 77.77777777777777,
        "score": 83.7872765059474
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 19,
        "noul": 9,
        "score": 9
       },
       "score": 87.18835142790839
      },
      "fi": {
       "accuracy": 0.9038461538461539,
       "cc": 86.81418698282376,
       "cc_by_type": {
        "choice": 86.28571428571429,
        "noul": 89.4736842105263,
        "score": 84.68316245223069
       },
       "low_n": false,
       "n": 52,
       "n_by_type": {
        "choice": 21,
        "noul": 19,
        "score": 12
       },
       "score": 86.81418698282376
      },
      "fr": {
       "accuracy": 0.84375,
       "cc": 75.3798607989936,
       "cc_by_type": {
        "choice": 69.38775510204081,
        "noul": 77.77777777777777,
        "score": 78.97404951716221
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 14,
        "noul": 9,
        "score": 9
       },
       "score": 75.3798607989936
      },
      "hi": {
       "accuracy": 0.8378378378378378,
       "cc": 73.26256574773736,
       "cc_by_type": {
        "choice": 83.47107438016529,
        "noul": 55.55555555555556,
        "score": 80.76106730749122
       },
       "low_n": false,
       "n": 37,
       "n_by_type": {
        "choice": 17,
        "noul": 9,
        "score": 11
       },
       "score": 73.26256574773736
      },
      "id": {
       "accuracy": 0.9512195121951219,
       "cc": 85.96393294173264,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 91.22513215853121
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 15,
        "noul": 12,
        "score": 14
       },
       "score": 85.96393294173264
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 86.87408780564378,
       "cc_by_type": {
        "choice": 100,
        "noul": 83.33333333333333,
        "score": 77.288930083598
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 16
       },
       "score": 86.87408780564378
      },
      "ja": {
       "accuracy": 0.9545454545454546,
       "cc": 89.8172182512476,
       "cc_by_type": {
        "choice": 100,
        "noul": 81.81818181818181,
        "score": 87.633472935561
       },
       "low_n": false,
       "n": 44,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 16
       },
       "score": 89.8172182512476
      },
      "ko": {
       "accuracy": 0.8604651162790697,
       "cc": 73.72602207821258,
       "cc_by_type": {
        "choice": 70.58823529411767,
        "noul": 60.00000000000001,
        "score": 90.58983094052007
       },
       "low_n": false,
       "n": 43,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 15
       },
       "score": 73.72602207821258
      },
      "mixed": {
       "accuracy": 0.90625,
       "cc": 86.39509022731563,
       "cc_by_type": {
        "choice": 78.3978397839784,
        "noul": 89.99999999999999,
        "score": 90.78743089796852
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 26,
        "noul": 20,
        "score": 18
       },
       "score": 86.39509022731563
      },
      "nl": {
       "accuracy": 0.8611111111111112,
       "cc": 57.11661111194095,
       "cc_by_type": {
        "choice": 85.84905660377359,
        "noul": 60.00000000000001,
        "score": 25.500776732049246
       },
       "low_n": false,
       "n": 36,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 6
       },
       "score": 57.11661111194095
      },
      "no": {
       "accuracy": 0.9019607843137255,
       "cc": 87.89279853464124,
       "cc_by_type": {
        "choice": 79.34595524956971,
        "noul": 86.66666666666667,
        "score": 97.66577368768733
       },
       "low_n": false,
       "n": 51,
       "n_by_type": {
        "choice": 27,
        "noul": 15,
        "score": 9
       },
       "score": 87.89279853464124
      },
      "pl": {
       "accuracy": 0.8771929824561403,
       "cc": 79.84292349121434,
       "cc_by_type": {
        "choice": 85.63065460351251,
        "noul": 88.23529411764706,
        "score": 65.66282175248344
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 28,
        "noul": 17,
        "score": 12
       },
       "score": 79.84292349121434
      },
      "pt": {
       "accuracy": 0.9148936170212766,
       "cc": 82.81507832192635,
       "cc_by_type": {
        "choice": 88.01996672212977,
        "noul": 66.66666666666667,
        "score": 93.75860157698257
       },
       "low_n": false,
       "n": 47,
       "n_by_type": {
        "choice": 23,
        "noul": 12,
        "score": 12
       },
       "score": 82.81507832192635
      },
      "sv": {
       "accuracy": 0.9565217391304348,
       "cc": 88.58348395327084,
       "cc_by_type": {
        "choice": 93.29608938547484,
        "noul": 83.33333333333333,
        "score": 89.12102914100437
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 21,
        "noul": 12,
        "score": 13
       },
       "score": 88.58348395327084
      },
      "tr": {
       "accuracy": 0.8913043478260869,
       "cc": 89.38332376866492,
       "cc_by_type": {
        "choice": 72.63397947548461,
        "noul": 100,
        "score": 95.51599183051015
       },
       "low_n": false,
       "n": 46,
       "n_by_type": {
        "choice": 20,
        "noul": 10,
        "score": 16
       },
       "score": 89.38332376866492
      },
      "uk": {
       "accuracy": 0.9464285714285714,
       "cc": 89.29702256260133,
       "cc_by_type": {
        "choice": 94.52054794520548,
        "noul": 85.71428571428572,
        "score": 87.65623402831278
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 26,
        "noul": 14,
        "score": 16
       },
       "score": 89.29702256260133
      },
      "zh": {
       "accuracy": 0.926829268292683,
       "cc": 91.65229823464345,
       "cc_by_type": {
        "choice": 87.60330578512396,
        "noul": 100,
        "score": 87.3535889188064
       },
       "low_n": false,
       "n": 41,
       "n_by_type": {
        "choice": 22,
        "noul": 8,
        "score": 11
       },
       "score": 91.65229823464345
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8973333333333333,
       "cc": 85.995897788241,
       "cc_by_type": {
        "choice": 85.995897788241
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 85.995897788241
      },
      "noul": {
       "accuracy": 0.8826666666666667,
       "cc": 76.53333333333335,
       "cc_by_type": {
        "noul": 76.53333333333335
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 76.53333333333335
      },
      "score": {
       "accuracy": 0.9013333333333333,
       "cc": 85.36412050331485,
       "cc_by_type": {
        "score": 85.36412050331485
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 85.36412050331485
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.875,
       "cc": 82.48862837069302,
       "cc_by_type": {
        "choice": 77.76130467012601,
        "noul": 71.42857142857142,
        "score": 98.27600901338164
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 31,
        "noul": 7,
        "score": 10
       },
       "score": 82.48862837069302
      },
      "code_linting": {
       "accuracy": 0.9710144927536232,
       "cc": 95.75721215964434,
       "cc_by_type": {
        "choice": 96.32578077158605,
        "noul": 100,
        "score": 90.945855707347
       },
       "low_n": false,
       "n": 69,
       "n_by_type": {
        "choice": 38,
        "noul": 12,
        "score": 19
       },
       "score": 95.75721215964434
      },
      "customer_support": {
       "accuracy": 0.8939393939393939,
       "cc": 74.73315193218161,
       "cc_by_type": {
        "choice": 96.309963099631,
        "noul": 33.33333333333333,
        "score": 94.5561593635805
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 38,
        "noul": 15,
        "score": 13
       },
       "score": 74.73315193218161
      },
      "demand_forecasting": {
       "accuracy": 0.5909090909090909,
       "cc": 16.480405220814227,
       "cc_by_type": {
        "choice": 54.510993176648974,
        "noul": -38.46153846153846,
        "score": 33.39176094733216
       },
       "low_n": false,
       "n": 66,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 23
       },
       "score": 16.480405220814227
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.8870967741935484,
       "cc": 80.39829948146182,
       "cc_by_type": {
        "choice": 87.33286418015481,
        "noul": 64.70588235294117,
        "score": 89.15615191128946
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 33,
        "noul": 17,
        "score": 12
       },
       "score": 80.39829948146182
      },
      "feature_extraction": {
       "accuracy": 0.9027777777777778,
       "cc": 82.26322672348725,
       "cc_by_type": {
        "choice": 81.15577889447238,
        "noul": 71.42857142857142,
        "score": 94.20532984741796
       },
       "low_n": false,
       "n": 72,
       "n_by_type": {
        "choice": 37,
        "noul": 14,
        "score": 21
       },
       "score": 82.26322672348725
      },
      "financial_crime": {
       "accuracy": 0.8888888888888888,
       "cc": 69.01058557570461,
       "cc_by_type": {
        "choice": 90.43062200956938,
        "noul": 45.45454545454546,
        "score": 71.14658926299902
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 44,
        "noul": 11,
        "score": 8
       },
       "score": 69.01058557570461
      },
      "gaming": {
       "accuracy": 0.8275862068965517,
       "cc": 71.61186026049774,
       "cc_by_type": {
        "choice": 80.87649402390439,
        "noul": 55.55555555555556,
        "score": 78.40353120203328
       },
       "low_n": false,
       "n": 58,
       "n_by_type": {
        "choice": 29,
        "noul": 9,
        "score": 20
       },
       "score": 71.61186026049774
      },
      "insurance_claims": {
       "accuracy": 0.9032258064516129,
       "cc": 83.20708045961267,
       "cc_by_type": {
        "choice": 82.1826280623608,
        "noul": 88.23529411764706,
        "score": 79.20331919883014
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 31,
        "noul": 17,
        "score": 14
       },
       "score": 83.20708045961267
      },
      "knowledge_graphs": {
       "accuracy": 0.9701492537313433,
       "cc": 95.72681536153648,
       "cc_by_type": {
        "choice": 95.50224887556223,
        "noul": 100,
        "score": 91.67819720904724
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 30,
        "noul": 13,
        "score": 24
       },
       "score": 95.72681536153648
      },
      "lead_generation": {
       "accuracy": 0.9333333333333333,
       "cc": 89.37586302940963,
       "cc_by_type": {
        "choice": 91.5373765867419,
        "noul": 82.85714285714285,
        "score": 93.73306964434413
       },
       "low_n": false,
       "n": 75,
       "n_by_type": {
        "choice": 33,
        "noul": 35,
        "score": 7
       },
       "score": 89.37586302940963
      },
      "legal_compliance": {
       "accuracy": 0.875,
       "cc": 83.6511918194825,
       "cc_by_type": {
        "choice": 69.90972918756269,
        "noul": 85.71428571428572,
        "score": 95.32956055659912
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 23,
        "noul": 14,
        "score": 19
       },
       "score": 83.6511918194825
      },
      "llm_guardrails": {
       "accuracy": 0.9859154929577465,
       "cc": 97.25644485019507,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 91.76933455058521
       },
       "low_n": false,
       "n": 71,
       "n_by_type": {
        "choice": 34,
        "noul": 26,
        "score": 11
       },
       "score": 97.25644485019507
      },
      "model_routing": {
       "accuracy": 0.9256756756756757,
       "cc": 86.72285015956966,
       "cc_by_type": {
        "choice": 93.76839189890946,
        "noul": 94.11764705882352,
        "score": 72.28251152097602
       },
       "low_n": false,
       "n": 148,
       "n_by_type": {
        "choice": 84,
        "noul": 34,
        "score": 30
       },
       "score": 86.72285015956966
      },
      "moderation_trust_safety": {
       "accuracy": 0.9322033898305084,
       "cc": 92.16767746474274,
       "cc_by_type": {
        "choice": 100,
        "noul": 88.23529411764706,
        "score": 88.26773827658114
       },
       "low_n": false,
       "n": 59,
       "n_by_type": {
        "choice": 21,
        "noul": 17,
        "score": 21
       },
       "score": 92.16767746474274
      },
      "other": {
       "accuracy": 0.8125,
       "cc": 82.96086008679606,
       "cc_by_type": {
        "choice": 60.04439511653718,
        "noul": 100,
        "score": 88.83818514385099
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 5
       },
       "score": 82.96086008679606
      },
      "recruiting": {
       "accuracy": 0.9692307692307692,
       "cc": 96.20024152375986,
       "cc_by_type": {
        "choice": 91.34823359769285,
        "noul": 100,
        "score": 97.2524909735867
       },
       "low_n": false,
       "n": 65,
       "n_by_type": {
        "choice": 32,
        "noul": 25,
        "score": 8
       },
       "score": 96.20024152375986
      },
      "risk_assessment": {
       "accuracy": 0.8333333333333334,
       "cc": 78.24737822668754,
       "cc_by_type": {
        "choice": 62.34309623430962,
        "noul": 84.61538461538463,
        "score": 87.7836538303684
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 22,
        "noul": 13,
        "score": 13
       },
       "score": 78.24737822668754
      },
      "scientific_discovery": {
       "accuracy": 0.7857142857142857,
       "cc": 50.921305613601646,
       "cc_by_type": {
        "choice": 85.27607361963189,
        "noul": -19.999999999999996,
        "score": 87.48784322117305
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 18,
        "noul": 10,
        "score": 14
       },
       "score": 50.921305613601646
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.57671015104647,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.73013045313941
       },
       "low_n": false,
       "n": 67,
       "n_by_type": {
        "choice": 14,
        "noul": 14,
        "score": 39
       },
       "score": 99.57671015104647
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      90.4722551877888,
      94.57139944769332
     ],
     "I": [
      74.19713740972021,
      82.38188139592285
     ],
     "capability": [
      82.7751662910963,
      88.05443724392144
     ],
     "composite": null,
     "seed": 16
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 94.3167842335138,
     "capability": 86.54920580003778,
     "intelligence": 78.78162736656175
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8973333333333333,
      "calibration": 92.2599176730872,
      "cc": 85.995897788241,
      "cc_by_type": {
       "choice": 85.995897788241
      },
      "cc_tiered": 85.79521359891642,
      "cc_tiered_open": 84.8046790723018,
      "cc_tiered_sealed": 86.78574812553104,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 85.995897788241
     },
     "noul": {
      "accuracy": 0.8826666666666667,
      "calibration": 94.06525019831905,
      "cc": 76.53333333333335,
      "cc_by_type": {
       "noul": 76.53333333333335
      },
      "cc_tiered": 73.04992318550501,
      "cc_tiered_open": 77.07729468599035,
      "cc_tiered_sealed": 69.02255168501965,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 76.53333333333335
     },
     "score": {
      "accuracy": 0.9013333333333333,
      "calibration": 96.62518482913518,
      "cc": 85.36412050331485,
      "cc_by_type": {
       "score": 85.36412050331485
      },
      "cc_tiered": 77.49974531526384,
      "cc_tiered_open": 68.62708752281769,
      "cc_tiered_sealed": 86.37240310770999,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 85.36412050331485
     }
    },
    "raw_on_A": {
     "calibration": 90.7095761686435,
     "capability": 83.57138271970345,
     "intelligence": 76.4331892707634
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.005333333333333301,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 73.59141925380933,
    "Emphasis on Accuracy 60:20:20": 75.58321677308784,
    "Emphasis on Cost 20:20:60": 68.18780226781061,
    "Emphasis on Speed 20:60:20": 77.70133138547168,
    "Intelligence only": 78.78162736656175
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "vLLM Semantic Router Team (vllm-sr)",
   "axes": {
    "calibration": 91.37586990638505,
    "cost": 57.27607555734765,
    "intelligence": 72.92342726693953,
    "speed": 87.00758628687251
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.03405766539472167,
      "ece_score": 93.18846692105566,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 91.94595402877492,
      "score_before_time_limit": 93.18846692105566,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.1466666666666666,
      "brier": 0.06360963348259217,
      "ece": 0.056426602196387884,
      "n": 375,
      "score": 88.24153460306523,
      "score_before_time_limit": 88.71467956072242,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.984,
      "interval90_width": 0.24200000000000008,
      "n": 375,
      "nrps": 0.04397667161509103,
      "score": 93.94012108731499,
      "score_before_time_limit": 94.69770270898688,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.031034637102585673
     }
    },
    "score": 91.37586990638505
   },
   "capability": 82.1496485866623,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11823.4995401 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=trunk-equivalent). The row's own scored cost basis: 771,190 input + 0 output tokens over 1,485 decisions = 519.31986532 tokens/decision. GPU-time cost USD 0.0265540528174136/1,000; old list-price cost USD 0.218114343434343/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0265540528174136/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.5463374
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.02655405281741362
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "vLLM-SR d3",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-12",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned rented Verda GPU VM, 1x NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (confidential-computing mode, attested), NOT the 1x H100 80 GB on which this draw's other GPU rows and the published Perplexity Decider and Torchcast-Decision-27B rows were timed - no single H100 was available at measurement time - so this row's Speed axis rests on different hardware from those rows (some other published v1.6 rows, for example xor-26b-a4b and rene-1-31b-fp8, were also timed on an RTX PRO 6000). The author's own unmodified d3_server.py (plain CUDA / PyTorch path; the author's AMD ROCm fast path does not apply on CUDA) in an offline container with --network none, Hub offline mode and all credentials unset, weights verified against the author's MODEL_MANIFEST sha256 list; driven by the unchanged pinned pod_driver_v16 with one official warm-up and a sequential scoring loop. The standard self-hosted x2 + 0.15 s own-server adjustment (an assumption, not measured) is applied on top of the raw p50/p95. Measured 12 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b); VM destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 78.42164858078797,
    "I_sealed": 67.42520595309108,
    "base": 72.92342726693953,
    "excess": 1.1569158573411826,
    "gap": 10.996442627696894,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 88.14184875359072,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 86.78829820698331,
       "judge": 88.13447593935399,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 77.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 50,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 70.04580230278285,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 66.43476019846001,
       "hard": 76.385216737932,
       "judge": 75.95310460653639,
       "standard": 50.31154102901563
      }
     },
     "sealed|choice": {
      "cc": 80.69575983859798,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 96.18805590851335,
       "hard": 71.89776801202729,
       "judge": 83.2517140058766,
       "standard": 86.71166420586374
      }
     },
     "sealed|noul": {
      "cc": 56.25893200983268,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 86.20689655172413,
       "hard": 43.636363636363626,
       "judge": 53.846153846153854,
       "standard": 70.1492537313433
      }
     },
     "sealed|score": {
      "cc": 65.32092601084263,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 84.15753121858664,
       "hard": 62.596109395819276,
       "judge": 61.28832264025312,
       "standard": 67.40116169290158
      }
     }
    }
   },
   "jevbench_score": 74.61382220232323,
   "key": "vllm-sr-d3",
   "last_measured_on": "2026-10-12",
   "licence": "Apache-2.0 (weights, model card and NOTICE; base Qwen/Qwen3.8-27B Apache-2.0). Training data not disclosed",
   "listing": "ranked",
   "measured_in": "v1.6.16",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "vllm-sr/d3@5ce995dfb7ff07da00fec4211eb7f0560f52eaf8",
   "noul_decisive": {
    "acc_among_decisive": 0.9623824451410659,
    "decisive": 319,
    "decisive_rate": 0.8552278820375335,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 92.20028306358832,
     "cost": 29.839473214267542,
     "intelligence": 73.73157285955872,
     "speed": 87.36893361347018
    },
    "board_release": "v1.6.43",
    "capability": 82.96592796157353,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.171426588851887,
      20.699584134625216
     ],
     "B": [
      20.946864446552823,
      21.780666403522886
     ],
     "C": [
      20.171426588851887,
      20.699584134625216
     ]
    },
    "jevbench_score": 20.53757421097071,
    "measured_in": "v1.6.16",
    "method": "v1.6",
    "open_board_rank": 94,
    "rank": 128,
    "ranks": {
     "A": 128,
     "B": 119,
     "C": 110,
     "capability": 11
    },
    "scores": {
     "A": 20.53757421097071,
     "B": 21.473456539895587,
     "C": 20.53757421097071
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.36749324100005654,
     "p50_s_raw": 0.10874662050002826,
     "p95_s_adjusted": 0.4987205717000961,
     "p95_s_raw": 0.17436028585004806
    },
    "views": {
     "Balanced 33:33:33": 18.257915673012125,
     "Emphasis on Accuracy 60:20:20": 20.792305962031904,
     "Emphasis on Cost 20:20:60": 14.18430475741736,
     "Emphasis on Speed 20:60:20": 21.87364349863337,
     "Intelligence only": 26.26006480299088
    }
   },
   "rank": 24,
   "ranked": true,
   "ranks": {
    "A": 24,
    "B": 17,
    "C": 21,
    "capability": 17
   },
   "repo": "https://huggingface.co/vllm-sr/d3",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.61382220232323,
    "B": 74.26950269919122,
    "C": 74.61382220232323
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3702694839998003,
    "p50_s_raw": 0.11013474199990014,
    "p95_s_adjusted": 0.5379271327999277,
    "p95_s_raw": 0.19396356639996384
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 79.70393292683865,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 47.44513211384924
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 79.70393292683865
      },
      "calibration_gen": {
       "accuracy": 0.7623762376237624,
       "cc": 61.321415531109665,
       "cc_by_type": {
        "choice": 58.642629904559925,
        "noul": 52,
        "score": 73.32161668876907
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.321415531109665
      },
      "composite_gen": {
       "accuracy": 0.9207920792079208,
       "cc": 88.97552391667175,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 80.83804419807305
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 88.97552391667175
      },
      "fairness": {
       "accuracy": 0.9375,
       "cc": 85.84803221342577,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 97.54409664027729
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 85.84803221342577
      },
      "jaggedness": {
       "accuracy": 0.7055555555555556,
       "cc": 54.01362633380514,
       "cc_by_type": {
        "choice": 63.90760346487007,
        "noul": 29.729729729729737,
        "score": 68.4035458068156
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 54.01362633380514
      },
      "llm_router": {
       "accuracy": 0.939297124600639,
       "cc": 85.14867822344807,
       "cc_by_type": {
        "choice": 96.03768600862904,
        "noul": 100,
        "score": 59.40834866171518
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 85.14867822344807
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6666666666666666,
       "cc": 40.30049701458755,
       "cc_by_type": {
        "choice": 62.886597938144334,
        "noul": 0,
        "score": 58.01489310561831
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 40.30049701458755
      },
      "multilingual": {
       "accuracy": 0.7890909090909091,
       "cc": 63.79693472202507,
       "cc_by_type": {
        "choice": 81.41506358004565,
        "noul": 44.44444444444444,
        "score": 65.53129614158513
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 63.79693472202507
      },
      "negation_pair_gen": {
       "accuracy": 0.9069767441860465,
       "cc": 78.16865484806344,
       "cc_by_type": {
        "choice": 100,
        "noul": 73.33333333333334,
        "score": 61.172631210856984
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 78.16865484806344
      },
      "option_order": {
       "accuracy": 0.9791666666666666,
       "cc": 97.3368841544607,
       "cc_by_type": {
        "choice": 97.3368841544607
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 97.3368841544607
      },
      "perturb_pair": {
       "accuracy": 0.90625,
       "cc": 86.04495127304953,
       "cc_by_type": {
        "choice": 87.98798798798799,
        "noul": 100,
        "score": 70.14686583116061
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 86.04495127304953
      },
      "usecase": {
       "accuracy": 0.7727272727272727,
       "cc": 61.382363634162466,
       "cc_by_type": {
        "choice": 77.94117647058823,
        "noul": 37.5,
        "score": 68.70591443189917
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 61.382363634162466
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.75,
       "cc": 69.16376241279302,
       "cc_by_type": {
        "choice": 80.95238095238095,
        "noul": 100,
        "score": 26.53890628599814
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 69.16376241279302
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.7692307692307693,
       "cc": 46.72533851096023,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 6.842682199547367
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 46.72533851096023
      },
      "de": {
       "accuracy": 0.875,
       "cc": 85.59895551835274,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 100,
        "score": 80.69067186479275
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 85.59895551835274
      },
      "el": {
       "accuracy": 0.6666666666666666,
       "cc": 55.49221077651324,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 0,
        "score": 99.9961854021654
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 55.49221077651324
      },
      "en": {
       "accuracy": 0.8427454387489139,
       "cc": 73.66051265515044,
       "cc_by_type": {
        "choice": 82.86572045740228,
        "noul": 68.34532374100719,
        "score": 69.77049376704187
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 73.66051265515044
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.27374935392363,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 29.60917628363061
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.27374935392363
      },
      "fi": {
       "accuracy": 0.875,
       "cc": 77.71287671897267,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 99.80529682358468
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 77.71287671897267
      },
      "fr": {
       "accuracy": 0.8571428571428571,
       "cc": 83.92352369573172,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 94.12351226366575
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 83.92352369573172
      },
      "hi": {
       "accuracy": 0.7619047619047619,
       "cc": 50.875059370970995,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 25,
        "score": 44.384954649225826
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 50.875059370970995
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.8,
       "cc": 73.4076618326374,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 34.25260358053105
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 73.4076618326374
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 38.966499106440274,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 16.899497319320812
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 38.966499106440274
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 19.173970994902465,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 57.52191298470741
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 19.173970994902465
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.92564862400029,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.77694587200085
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.92564862400029
      },
      "nl": {
       "accuracy": 0.5,
       "cc": -20.032834215525288,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -100,
        "score": -34.124476672549896
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8461538461538461,
       "cc": 79.398514682308,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 80,
        "score": 65.81297951031802
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 79.398514682308
      },
      "pt": {
       "accuracy": 0.8571428571428571,
       "cc": 79.23908352196145,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 69.53811090766393
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 79.23908352196145
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 47.20572405326843,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 19.999999999999996,
        "score": 47.149087053422335
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 47.20572405326843
      },
      "tr": {
       "accuracy": 0.9230769230769231,
       "cc": 85.40756411534664,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 56.22269234603995
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 85.40756411534664
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 78.18093252705013,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 100,
        "score": 99.05892661340849
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 78.18093252705013
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 48.40707026933146,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 11.887877474661067
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 48.40707026933146
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8653333333333333,
       "cc": 82.60586693201171,
       "cc_by_type": {
        "choice": 82.60586693201171
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 82.60586693201171
      },
      "noul": {
       "accuracy": 0.8186666666666667,
       "cc": 63.733333333333334,
       "cc_by_type": {
        "noul": 63.733333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 63.733333333333334
      },
      "score": {
       "accuracy": 0.7706666666666667,
       "cc": 67.41226190218106,
       "cc_by_type": {
        "score": 67.41226190218106
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 67.41226190218106
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5454545454545454,
       "cc": 19.11264958464173,
       "cc_by_type": {
        "choice": 84,
        "noul": -50,
        "score": 23.337948753925186
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 19.11264958464173
      },
      "code_linting": {
       "accuracy": 0.9166666666666666,
       "cc": 89.26555045427034,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 67.79665136281103
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 89.26555045427034
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 34.48589320988469,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 50,
        "score": 2.2381674345321345
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 34.48589320988469
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -1.3162243525891217,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -67.10656779460946
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 39.51821695055512,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 80.8734914313755
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 39.51821695055512
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 40.793860182167386,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": 1.6268635653700847
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 40.793860182167386
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 53.855675565990985,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 66.88915554951359
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 53.855675565990985
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 70.98778127144608,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50,
        "score": 73.4163054868121
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 70.98778127144608
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 14.027076831306962,
       "cc_by_type": {
        "choice": 37.5,
        "noul": -60,
        "score": 64.58123049392088
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 14.027076831306962
      },
      "knowledge_graphs": {
       "accuracy": 0.9090909090909091,
       "cc": 79.02166050259233,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 70.39831484111032
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 79.02166050259233
      },
      "lead_generation": {
       "accuracy": 0.76,
       "cc": 70.03571035914229,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 77.77777777777777,
        "score": 43.957260276393285
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 70.03571035914229
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 17.71047097327813,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 48.830337651017196
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 17.71047097327813
      },
      "llm_guardrails": {
       "accuracy": 0.8518518518518519,
       "cc": 61.7232011127565,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 65.16960333826947
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 61.7232011127565
      },
      "model_routing": {
       "accuracy": 0.9078341013824884,
       "cc": 79.70594033783782,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 96.96969696969697,
        "score": 46.82344871914113
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 79.70594033783782
      },
      "moderation_trust_safety": {
       "accuracy": 0.6111111111111112,
       "cc": 13.96550875487595,
       "cc_by_type": {
        "choice": 65.4510556621881,
        "noul": -100,
        "score": 76.44547060243976
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 13.96550875487595
      },
      "recruiting": {
       "accuracy": 0.7777777777777778,
       "cc": 59.29379240056826,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 33.33333333333333,
        "score": 77.97422771238257
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 59.29379240056826
      },
      "risk_assessment": {
       "accuracy": 0.6666666666666666,
       "cc": 37.6695454341995,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": 0,
        "score": 27.7869614257512
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 37.6695454341995
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 28.5849244629175,
       "cc_by_type": {
        "choice": 42.02898550724638,
        "noul": -33.333333333333336,
        "score": 77.05912121483945
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 28.5849244629175
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.92396962090108,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.77190886270323
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.92396962090108
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.99802173783495,
      91.86975408101644
     ],
     "I": [
      68.29342272755383,
      76.59687229252359
     ],
     "capability": [
      78.2543213807504,
      83.72614571299034
     ],
     "composite": null,
     "seed": 83
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 91.37586990638505,
     "capability": 82.1496485866623,
     "intelligence": 72.92342726693953
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8653333333333333,
      "calibration": 91.94595402877492,
      "cc": 82.60586693201171,
      "cc_by_type": {
       "choice": 82.60586693201171
      },
      "cc_tiered": 84.41880429609435,
      "cc_tiered_open": 88.14184875359072,
      "cc_tiered_sealed": 80.69575983859798,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 82.60586693201171
     },
     "noul": {
      "accuracy": 0.8186666666666667,
      "calibration": 88.24153460306523,
      "cc": 63.733333333333334,
      "cc_by_type": {
       "noul": 63.733333333333334
      },
      "cc_tiered": 66.66811334791151,
      "cc_tiered_open": 77.07729468599035,
      "cc_tiered_sealed": 56.25893200983268,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 63.733333333333334
     },
     "score": {
      "accuracy": 0.7706666666666667,
      "calibration": 93.94012108731499,
      "cc": 67.41226190218106,
      "cc_by_type": {
       "score": 67.41226190218106
      },
      "cc_tiered": 67.68336415681273,
      "cc_tiered_open": 70.04580230278285,
      "cc_tiered_sealed": 65.32092601084263,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 67.41226190218106
     }
    },
    "raw_on_A": {
     "calibration": 89.95256547604913,
     "capability": 80.86122053647321,
     "intelligence": 71.7698755968973
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.31432094739169,
    "Emphasis on Accuracy 60:20:20": 71.3352340722954,
    "Emphasis on Cost 20:20:60": 64.44614864475382,
    "Emphasis on Speed 20:60:20": 76.1590689778114,
    "Intelligence only": 72.92342726693953
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "vLLM Semantic Router Team (vllm-sr)",
   "axes": {
    "calibration": 86.75010672241689,
    "cost": 78.76524306413529,
    "intelligence": 50.78521313506569,
    "speed": 93.06198600681341
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.058056764953574355,
      "ece_score": 88.38864700928512,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 87.21013171582798,
      "score_before_time_limit": 88.38864700928512,
      "time_limit_failed_share": 0.013333333333333334
     },
     "noul": {
      "abstention_rate": 0.22666666666666668,
      "brier": 0.11893480874288055,
      "ece": 0.06490408428587044,
      "n": 375,
      "score": 86.55508083273084,
      "score_before_time_limit": 87.01918314282591,
      "time_limit_failed_share": 0.005333333333333333
     },
     "score": {
      "interval90_coverage": 0.936,
      "interval90_width": 0.36888888888888893,
      "n": 375,
      "nrps": 0.08737047233013917,
      "score": 86.48510761869188,
      "score_before_time_limit": 87.18256816400391,
      "time_limit_failed_share": 0.008,
      "top_ece": 0.08448908219489133
     }
    },
    "score": 86.75010672241689
   },
   "capability": 68.7676599287413,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 771,190 input + 0 output tokens over 1,485 decisions = 519.31986532 tokens/decision. GPU-time cost USD 0.0051029960386139/1,000; old list-price cost USD 0.015579595959596/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0051029960386139/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.0390241
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.0051029960386139005
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 15,
    "n_over_limit_answered_ok": 15,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "vLLM-SR d3-mini",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB, FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5991917808219176/h recorded by the runner (300 GB disk). Same GPU class as the published d3 and d3-turbo rows; hardware/mode conditions differ across the wider field. The author's own unmodified stdlib server on loopback in a --network none --read-only container; serial requests, read-only code/model/input, empty credential variables and HF offline flags. In-container network cut proven before sealed input; all 22 snapshot files re-hashed, offline flag values and absence of author-path override variables checked. The standard self-hosted x2 + 0.15 s own-server adjustment is an assumption, not measured. Measured 24 Oct 2026, pod window 09:07-09:19Z, approximately USD 0.50 rental. Measurement container 307 s, run cap 2856 s, complete=true, deadline_hit=false, rc 0; scored loop 224.870576 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 814d4afdcc638f3773e3b6263f98e1319e952a2115e6ff1f3cca6c18e98f88af. Node-shred and row cleanup verified; exact-id VM and volume teardown recorded, provider absent-or-discontinued and guard closed. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 56.122863162304306,
    "I_sealed": 45.44756310782708,
    "base": 50.78521313506569,
    "excess": 0.8357732841215153,
    "gap": 10.675300054477226,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.89077514236276,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 66.02705253224286,
       "judge": 68.35860250494397,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 46.39406487232574,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 21.739130434782616,
       "judge": 50,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 51.083749472224426,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 56.81334321079525,
       "hard": 50.57315275281009,
       "judge": 50.10563825926111,
       "standard": 50.70731286121263
      }
     },
     "sealed|choice": {
      "cc": 63.956026815917554,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 75.22236340533672,
       "hard": 57.32624031455995,
       "judge": 62.09598432908915,
       "standard": 74.3724952541658
      }
     },
     "sealed|noul": {
      "cc": 31.726758059233617,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 65.51724137931035,
       "hard": 21.818181818181827,
       "judge": 19.999999999999996,
       "standard": 52.23880597014925
      }
     },
     "sealed|score": {
      "cc": 40.65990444833008,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 58.88216493509184,
       "hard": 41.7254986343324,
       "judge": 30.53343502273288,
       "standard": 44.607289971340336
      }
     }
    }
   },
   "jevbench_score": 73.18017064643449,
   "key": "vllm-sr-d3-mini",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 at the measured pin (model card, LICENSE and NOTICE); base Qwen/Qwen3.5-4B Apache-2.0; public, not gated. Training data not disclosed",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "vllm-sr/d3-mini@bd0090dd758d32ba59ef5c749910a8e87fc4391f",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.9097222222222222,
    "decisive": 288,
    "decisive_rate": 0.7721179624664879,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 46,
   "ranked": true,
   "ranks": {
    "A": 46,
    "B": 76,
    "C": 98,
    "capability": 109
   },
   "repo": "https://huggingface.co/vllm-sr/d3-mini",
   "scores": {
    "A": 73.18017064643449,
    "B": 67.2491459513127,
    "C": 52.428263799877506
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20653101700005436,
    "p50_s_raw": 0.02826550850002718,
    "p95_s_adjusted": 0.2392302618999281,
    "p95_s_raw": 0.044615130949964055
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN ENGINE, run unmodified: upstream vllm-sr/d3-mini at bd0090dd758d32ba59ef5c749910a8e87fc4391f, the author's stdlib d3_server.py with d3_runtime.py, d3_format.py, d3_engine.py, d3_fast.py and d3_kernels.py. The measured repository is vllm-sr/d3-mini; zils-ai/z3-mini-v1 and zils-ai/d3-mini-4.5b-v1 are byte-identical weight mirrors, not separate measurements. The server, engine and kernels are byte-identical to the reviewed d3-turbo pin 689b3af198baaf6f11110be38a9daa7188dfd8ad; d3_runtime.py and d3_format.py are the older non-causal-only variants, reviewed separately. The d3-mini weight shards/readout share 0 LFS sha256 objects with the measured d3 and d3-turbo siblings (r25-card/sibling-weight-distinct-20261024.txt). Author defaults: --batch-size 8, --verify fast, warm-up on, device cuda:0, no input-length cap; no evaluator launcher, shim, patch or flag changes the model path. bf16 Qwen3.5-4B backbone, noncausal_full_attention (8 full-attention layers bidirectional, 24 Gated-DeltaNet layers causal), SDPA, plain CUDA / PyTorch path; the author's ROCm fast path does not apply on CUDA. fp32 255-way readout, temperature 1.0, permutation_average off, one forward pass per question, no generation. Python stack: torch 2.13.0, transformers 5.17.0, tokenizers 0.23.2, safetensors 0.8.0, accelerate 1.15.0, fla-core 0.5.2 and flash-linear-attention 0.5.2. Base image vllm/vllm-openai:v0.31.0@sha256:a4a4c0437bf7240089da5f08aa370c4aee17ae5290f7a3b468825ee26c4c3a6b; derived image sha256:c22d99ab6aca5d4f947d311da6af754a1d17b9183da92e85cd8a39e66c7fa057, hash-pinned wheel overlay installed offline. vLLM is in the image, not the serving backend. All 22 pinned snapshot files hash-checked in the measuring container; author MODEL_MANIFEST.json sha256 bc638e17982411f6ad0f524d33d0dc1f0db81ad7995c1bfdf93dba7674df5e48. The submitted code was reviewed and cleared at the exact pin; evaluator exposure, the device, pricing and context settings were cleared before the run. The source was reviewed with required safeguards; the runner was reviewed and findings fixed, with cold nonce probes and a maximum-of-repetitions fit check applied before the final review. pre-registration-D3MINI.md Amendments 1 and 2, including the token correction, govern this run. Weights: 4,539,265,536 bf16 backbone parameters plus fp32 readout; two weight shards plus readout.safetensors total 9,081,227,360 bytes. FIRST MEASUREMENT of vLLM-SR d3-mini (our own intake). AUTHOR'S OWN ENGINE on our rented hardware, one request at a time; the unchanged pinned pod_driver_v16 and TypeSafeAdapter drive the author's server. NATIVE OUTPUTS: choice, noul and score probabilities; no generated answer text. AUTHOR EXPOSURE: training data not disclosed by the card; no overlap or contamination audit was run by us. The author's Decision Index figures are a different measurement and are not independently verified by us. CONTEXT: as shipped, no input-length cap (decision_config.json max_length null and /health max_input_tokens null), no lower evaluator-imposed cap and no truncation; all 15 standard long items attempted, as required by ruling on file point 4. This differs from the published d3-turbo row, measured with its author's 32,768-token default cap; the published d3 row also had no cap. All 1,500 items answered HTTP 200, 0 refusals, preregistered class A_all_answered. 10 s RULE: exactly 15 decisions over 10 s, the standard long items at positions in the pre-registered long-item set, taking 10.543-17.503 s; these count wrong in the v1.7 share view (frozen v1.6 and v1.7-off views apply no time limit). Maximum 17.503080 s, median 0.028272 s, p95 0.044697 s. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. This is a distinct checkpoint from d3 and d3-turbo, with no shared LFS weight shards/readout; the zils-ai mirrors are not additional entries.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 40.63164823450948,
       "cc_by_type": {
        "choice": 66.66666666666667,
        "noul": 100,
        "score": -44.771721963138255
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 40.63164823450948
      },
      "calibration_gen": {
       "accuracy": 0.5841584158415841,
       "cc": 37.73069621287146,
       "cc_by_type": {
        "choice": 39.55461293743373,
        "noul": 43.99999999999999,
        "score": 29.637475701180648
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 37.73069621287146
      },
      "composite_gen": {
       "accuracy": 0.8316831683168316,
       "cc": 73.75553872088415,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 70.37037037037037,
        "score": 67.15640839390811
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 73.75553872088415
      },
      "fairness": {
       "accuracy": 0.75,
       "cc": 63.10908224458594,
       "cc_by_type": {
        "choice": 100,
        "noul": 19.999999999999996,
        "score": 69.32724673375782
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 63.10908224458594
      },
      "jaggedness": {
       "accuracy": 0.55,
       "cc": 28.69703799210934,
       "cc_by_type": {
        "choice": 39.364773820981725,
        "noul": 13.513513513513509,
        "score": 33.21282664183279
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 28.69703799210934
      },
      "llm_router": {
       "accuracy": 0.7955271565495208,
       "cc": 67.03168558806748,
       "cc_by_type": {
        "choice": 75.43365325350004,
        "noul": 71.13402061855669,
        "score": 54.5273828921457
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 67.03168558806748
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.4074074074074074,
       "cc": -9.927812416090854,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -75,
        "score": 19.443366875438784
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 0
      },
      "multilingual": {
       "accuracy": 0.6836363636363636,
       "cc": 45.17330606603309,
       "cc_by_type": {
        "choice": 68.6990544506032,
        "noul": 16.666666666666675,
        "score": 50.15419708082939
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 45.17330606603309
      },
      "negation_pair_gen": {
       "accuracy": 0.8255813953488372,
       "cc": 61.06174933224694,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 46.66666666666666,
        "score": 43.18524799674084
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 61.06174933224694
      },
      "option_order": {
       "accuracy": 0.8541666666666666,
       "cc": 81.35818908122504,
       "cc_by_type": {
        "choice": 81.35818908122504
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 81.35818908122504
      },
      "perturb_pair": {
       "accuracy": 0.890625,
       "cc": 84.510651461271,
       "cc_by_type": {
        "choice": 84.98498498498498,
        "noul": 100,
        "score": 68.54696939882801
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 84.510651461271
      },
      "usecase": {
       "accuracy": 0.6,
       "cc": 40.20280485542287,
       "cc_by_type": {
        "choice": 60.78431372549021,
        "noul": 20.833333333333325,
        "score": 38.99076750744508
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 40.20280485542287
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.4375,
       "cc": -9.311607822425833,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": -50,
        "score": -20.79196632442035
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 0
      },
      "cs": {
       "accuracy": 0.42857142857142855,
       "cc": 57.89473684210526,
       "cc_by_type": {
        "choice": 15.789473684210522,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 57.89473684210526
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 35.36820231523675,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": 33.33333333333333,
        "score": -6.491399198683001
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 35.36820231523675
      },
      "de": {
       "accuracy": 0.75,
       "cc": 73.36610796233718,
       "cc_by_type": {
        "choice": 52.21238938053097,
        "noul": 100,
        "score": 67.88593450648057
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 73.36610796233718
      },
      "el": {
       "accuracy": 0.4666666666666667,
       "cc": 27.681387194716347,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -66.66666666666667,
        "score": 99.99015785975425
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 27.681387194716347
      },
      "en": {
       "accuracy": 0.6993918331885317,
       "cc": 51.59869853180825,
       "cc_by_type": {
        "choice": 65.95111116535065,
        "noul": 45.32374100719425,
        "score": 43.52124342287985
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 51.59869853180825
      },
      "es": {
       "accuracy": 0.7741935483870968,
       "cc": 62.08157234583805,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 5.820573481233538
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 62.08157234583805
      },
      "fi": {
       "accuracy": 0.5,
       "cc": 44.736663698553684,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 66.45042825413101
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 44.736663698553684
      },
      "fr": {
       "accuracy": 0.7857142857142857,
       "cc": 77.01958842781792,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 73.41170645992435
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 77.01958842781792
      },
      "hi": {
       "accuracy": 0.6190476190476191,
       "cc": 29.535686556230626,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": -25,
        "score": 63.886389277630414
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 29.535686556230626
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 80.31764827211875,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 54.9825628989751
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 80.31764827211875
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 36.165394585019165,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 39.99999999999999,
        "score": 11.38736723400906
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 36.165394585019165
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 1.193668607293527,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": -100,
        "score": 48.02545026632503
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 1.193668607293527
      },
      "mixed": {
       "accuracy": 0.9285714285714286,
       "cc": 76.36440108052369,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 95.75986990823773
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 76.36440108052369
      },
      "nl": {
       "accuracy": 0.6,
       "cc": -2.357836041615627,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": -7.073508124846883
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.3333333333333333,
       "cc": -7.258064516129032,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.7435897435897436,
       "cc": 56.11967176332876,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 63.593886216774244
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 56.11967176332876
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 56.672476568725976,
       "cc_by_type": {
        "choice": 57.57218621096051,
        "noul": 60.00000000000001,
        "score": 52.44524349521742
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 56.672476568725976
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 48.98508227773069,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 19.999999999999996,
        "score": 52.48716172680909
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 48.98508227773069
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 42.022185067124546,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": -23.933444798626358
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 42.022185067124546
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 69.61883570307816,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 41.114571625363524
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 69.61883570307816
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 47.94274705325116,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 10.494907826420164
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 47.94274705325116
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.74,
       "cc": 66.41726783903252,
       "cc_by_type": {
        "choice": 66.41726783903252
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 66.41726783903252
      },
      "noul": {
       "accuracy": 0.6986666666666667,
       "cc": 39.733333333333334,
       "cc_by_type": {
        "noul": 39.733333333333334
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 39.733333333333334
      },
      "score": {
       "accuracy": 0.6053333333333333,
       "cc": 44.100372801134455,
       "cc_by_type": {
        "score": 44.100372801134455
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 44.100372801134455
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.45454545454545453,
       "cc": 6.389036030080741,
       "cc_by_type": {
        "choice": 68,
        "noul": -50,
        "score": 1.1671080902422193
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 6.389036030080741
      },
      "code_linting": {
       "accuracy": 0.4583333333333333,
       "cc": 14.76164364876596,
       "cc_by_type": {
        "choice": 25.465838509316768,
        "noul": 11.111111111111116,
        "score": 7.707981325869994
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 14.76164364876596
      },
      "customer_support": {
       "accuracy": 0.6428571428571429,
       "cc": 39.51825430650474,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 0,
        "score": 67.33525072439225
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 39.51825430650474
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": -27.57346684962377,
       "cc_by_type": {
        "choice": 18.128654970760234,
        "noul": -33.333333333333336,
        "score": -67.5157221862982
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.5882352941176471,
       "cc": 21.618848047320437,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -33.333333333333336,
        "score": 41.66813834485987
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 21.618848047320437
      },
      "feature_extraction": {
       "accuracy": 0.5263157894736842,
       "cc": -56.86789857623726,
       "cc_by_type": {
        "choice": 37.106918238993714,
        "noul": 33.33333333333333,
        "score": -241.04394730103883
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6,
       "cc": 38.30068301713463,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 20.22417790294452
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 38.30068301713463
      },
      "gaming": {
       "accuracy": 0.45454545454545453,
       "cc": 10.160005486264879,
       "cc_by_type": {
        "choice": 47.73519163763067,
        "noul": -50,
        "score": 32.74482482116397
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 10.160005486264879
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 2.534002247445445,
       "cc_by_type": {
        "choice": 16.666666666666668,
        "noul": -60,
        "score": 50.935340075669664
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 2.534002247445445
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 42.47689290860803,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 45.48920834752287
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 42.47689290860803
      },
      "lead_generation": {
       "accuracy": 0.56,
       "cc": 28.530781387905918,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": -11.111111111111116,
        "score": 19.959269228317243
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 28.530781387905918
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 12.886497100776092,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 34.35841603351108
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 12.886497100776092
      },
      "llm_guardrails": {
       "accuracy": 0.7037037037037037,
       "cc": 55.97192219635028,
       "cc_by_type": {
        "choice": 100.00000000000001,
        "noul": 19.999999999999996,
        "score": 47.915766589050826
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 55.97192219635028
      },
      "model_routing": {
       "accuracy": 0.8433179723502304,
       "cc": 70.69713565968036,
       "cc_by_type": {
        "choice": 91.81818181818183,
        "noul": 78.78787878787878,
        "score": 41.485346372980445
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 70.69713565968036
      },
      "moderation_trust_safety": {
       "accuracy": 0.7222222222222222,
       "cc": 11.277283989001177,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": -100,
        "score": 56.864481525544775
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 11.277283989001177
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 56.91434168592918,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 54.122783646459794
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 56.91434168592918
      },
      "risk_assessment": {
       "accuracy": 0.5555555555555556,
       "cc": 18.561851789482514,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50,
        "score": 20.46388049160025
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 18.561851789482514
      },
      "scientific_discovery": {
       "accuracy": 0.43478260869565216,
       "cc": 10.043399462512967,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 35.927299836814264
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 10.043399462512967
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 98.39989987588321,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 95.19969962764965
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 98.39989987588321
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.15681201246277,
      87.71391054836684
     ],
     "I": [
      44.60747334361465,
      55.13127592033528
     ],
     "capability": [
      63.96092710139239,
      70.47357791075639
     ],
     "composite": null,
     "seed": 84
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.75010672241689,
     "capability": 68.7676599287413,
     "intelligence": 50.78521313506569
    },
    "per_type": {
     "choice": {
      "accuracy": 0.74,
      "calibration": 87.21013171582798,
      "cc": 66.41726783903252,
      "cc_by_type": {
       "choice": 66.41726783903252
      },
      "cc_tiered": 67.42340097914015,
      "cc_tiered_open": 70.89077514236276,
      "cc_tiered_sealed": 63.956026815917554,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 66.41726783903252
     },
     "noul": {
      "accuracy": 0.6986666666666667,
      "calibration": 86.55508083273084,
      "cc": 39.733333333333334,
      "cc_by_type": {
       "noul": 39.733333333333334
      },
      "cc_tiered": 39.06041146577968,
      "cc_tiered_open": 46.39406487232574,
      "cc_tiered_sealed": 31.726758059233617,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.733333333333334
     },
     "score": {
      "accuracy": 0.6053333333333333,
      "calibration": 86.48510761869188,
      "cc": 44.100372801134455,
      "cc_by_type": {
       "score": 44.100372801134455
      },
      "cc_tiered": 45.871826960277254,
      "cc_tiered_open": 51.083749472224426,
      "cc_tiered_sealed": 40.65990444833008,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 44.100372801134455
     }
    },
    "raw_on_A": {
     "calibration": 87.76116280589144,
     "capability": 69.60593042154665,
     "intelligence": 51.45069803720186
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.5535206267242,
    "Emphasis on Accuracy 60:20:20": 60.595928805364196,
    "Emphasis on Cost 20:20:60": 72.96696227925746,
    "Emphasis on Speed 20:60:20": 77.37146897658907,
    "Intelligence only": 50.78521313506569
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "vLLM Semantic Router Team (vllm-sr)",
   "axes": {
    "calibration": 90.35330442455489,
    "cost": 57.25589630946454,
    "intelligence": 74.82897422466625,
    "speed": 85.64377521409548
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05697692176294583,
      "ece_score": 88.60461564741084,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 88.60461564741084,
      "score_before_time_limit": 88.60461564741084,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.13404825737265413,
      "brier": 0.0619027410142049,
      "ece": 0.061196749215429905,
      "n": 373,
      "score": 87.76065015691402,
      "score_before_time_limit": 87.76065015691402,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.946236559139785,
      "interval90_width": 0.1843637992831541,
      "n": 372,
      "nrps": 0.04245289733343758,
      "score": 94.6946474693398,
      "score_before_time_limit": 94.6946474693398,
      "time_limit_failed_share": 0,
      "top_ece": 0.03182707663988325
     }
    },
    "score": 90.35330442455489
   },
   "capability": 82.59113932461057,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: dense qwen, 27B active parameters; throughput 11805.2013064 tokens/s (direct measurement, 29 Oct 2026, stock vLLM 0.32.0 EFF, fresh 200 public decisions; load_mode=trunk-equivalent). The row's own scored cost basis: 771,190 input + 0 output tokens over 1,485 decisions = 519.31986532 tokens/decision. GPU-time cost USD 0.0265952119855034/1,000; old list-price cost USD 0.218114343434343/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0265952119855034/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.21593320000000002
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.026595211985503357
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "vLLM-SR d3-turbo",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-19",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.43",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned rented Verda GPU VM, 1x NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (FIN, pay-as-you-go non-spot), the same GPU class the published vllm-sr-d3 row was timed on (that row ran in confidential-computing mode, this one did not; raw p50 0.144 s vs d3's 0.109 s). The author's own unmodified stdlib d3_server.py (plain CUDA / PyTorch path; the author's ROCm fast path does not apply on CUDA) in an offline container with --network none (cut proven from inside the measuring container before the sealed input: loopback only, no default route, TCP and DNS fail), Hub offline mode and all credentials unset, every weight shard, readout, tokenizer, config and code file sha256-verified against the author's MODEL_MANIFEST.json; driven by the unchanged pinned pod_driver_v16. The standard self-hosted x2 + 0.15 s adjustment applies. Measured 19 Oct 2026 on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720...); determinism 50/50 bit-identical; VM and volume destroyed afterwards.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 79.94726417219914,
    "I_sealed": 69.71068427713337,
    "base": 74.82897422466625,
    "excess": 0.3970531247100624,
    "gap": 10.236579895065773,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 87.81849238331058,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 83.01352626612143,
       "judge": 92.089650626236,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 80.07729468599035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 91.30434782608697,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 71.9460054472965,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 74.70222801620412,
       "hard": 77.18034928834129,
       "judge": 73.57972551845668,
       "standard": 57.6486263740128
      }
     },
     "sealed|choice": {
      "cc": 81.76677045405546,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 86.6581956797967,
       "hard": 74.49982652943217,
       "judge": 85.89618021547503,
       "standard": 87.66083104830204
      }
     },
     "sealed|noul": {
      "cc": 58.81486706808374,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 79.3103448275862,
       "hard": 47.272727272727266,
       "judge": 53.846153846153854,
       "standard": 79.10447761194031
      }
     },
     "sealed|score": {
      "cc": 68.55041530926088,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 84.70603996671883,
       "hard": 64.62159006380274,
       "judge": 68.07303559090225,
       "standard": 69.04632304898617
      }
     }
    }
   },
   "jevbench_score": 74.66420728011205,
   "key": "vllm-sr-d3-turbo",
   "last_measured_on": "2026-10-19",
   "licence": "Apache-2.0 (weights, model card and NOTICE; base Qwen/Qwen3.8-27B family Apache-2.0). Training data not disclosed",
   "listing": "ranked",
   "measured_in": "v1.6.31",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "vllm-sr/d3-turbo@689b3af198baaf6f11110be38a9daa7188dfd8ad",
   "noul_decisive": {
    "acc_among_decisive": 0.9628482972136223,
    "decisive": 323,
    "decisive_rate": 0.8659517426273459,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 90.35330442455489,
     "cost": 29.839473214267542,
     "intelligence": 74.82897422466625,
     "speed": 86.03728492328564
    },
    "board_release": "v1.6.43",
    "capability": 82.59113932461057,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      20.127911685633038,
      20.65714979248832
     ],
     "B": [
      20.956179967315336,
      21.775300465066284
     ],
     "C": [
      20.127911685633038,
      20.65714979248832
     ]
    },
    "jevbench_score": 20.47854238643572,
    "measured_in": "v1.6.31",
    "method": "v1.6",
    "open_board_rank": 95,
    "rank": 129,
    "ranks": {
     "A": 129,
     "B": 120,
     "C": 111,
     "capability": 13
    },
    "scores": {
     "A": 20.47854238643572,
     "B": 21.473183288464952,
     "C": 20.47854238643572
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4387955279999233,
     "p50_s_raw": 0.14439776399996163,
     "p95_s_adjusted": 0.5675567784000123,
     "p95_s_raw": 0.20877838920000613
    },
    "views": {
     "Balanced 33:33:33": 18.264704402771564,
     "Emphasis on Accuracy 60:20:20": 20.894663067289894,
     "Emphasis on Cost 20:20:60": 14.186762674816508,
     "Emphasis on Speed 20:60:20": 21.784657900798422,
     "Intelligence only": 26.65091271040641
    }
   },
   "rank": 23,
   "ranked": true,
   "ranks": {
    "A": 23,
    "B": 15,
    "C": 20,
    "capability": 15
   },
   "repo": "https://huggingface.co/vllm-sr/d3-turbo",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.66420728011205,
    "B": 74.69710259503293,
    "C": 74.66420728011205
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.45193122999992286,
    "p50_s_raw": 0.15096561499996142,
    "p95_s_adjusted": 0.6033233377001693,
    "p95_s_raw": 0.22666166885008465
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.9545454545454546,
       "cc": 94.95104382165277,
       "cc_by_type": {
        "choice": 91.66666666666667,
        "noul": 100,
        "score": 93.18646479829165
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 94.95104382165277
      },
      "calibration_gen": {
       "accuracy": 0.7722772277227723,
       "cc": 60.4811415272761,
       "cc_by_type": {
        "choice": 68.18663838812301,
        "noul": 43.99999999999999,
        "score": 69.2567861937053
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 60.4811415272761
      },
      "composite_gen": {
       "accuracy": 0.9306930693069307,
       "cc": 90.19347841839642,
       "cc_by_type": {
        "choice": 93.4959349593496,
        "noul": 92.59259259259258,
        "score": 84.4919077032471
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 90.19347841839642
      },
      "fairness": {
       "accuracy": 1,
       "cc": 99.81665420977471,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.44996262932413
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 99.81665420977471
      },
      "jaggedness": {
       "accuracy": 0.7277777777777777,
       "cc": 56.78661196056641,
       "cc_by_type": {
        "choice": 62.463907603464875,
        "noul": 35.13513513513513,
        "score": 72.76079314309922
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 56.78661196056641
      },
      "llm_router": {
       "accuracy": 0.9297124600638977,
       "cc": 84.48235737075346,
       "cc_by_type": {
        "choice": 96.83014880690322,
        "noul": 95.8762886597938,
        "score": 60.74063464556334
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 84.48235737075346
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.7407407407407407,
       "cc": 44.32687494250607,
       "cc_by_type": {
        "choice": 75.2577319587629,
        "noul": -25,
        "score": 82.72289286875534
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 44.32687494250607
      },
      "multilingual": {
       "accuracy": 0.8254545454545454,
       "cc": 67.1694662456961,
       "cc_by_type": {
        "choice": 83.37137267688294,
        "noul": 50,
        "score": 68.13702606020536
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 67.1694662456961
      },
      "negation_pair_gen": {
       "accuracy": 0.8953488372093024,
       "cc": 75.39042370786824,
       "cc_by_type": {
        "choice": 100,
        "noul": 73.33333333333334,
        "score": 52.83793779027138
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 75.39042370786824
      },
      "option_order": {
       "accuracy": 0.9375,
       "cc": 92.01065246338216,
       "cc_by_type": {
        "choice": 92.01065246338216
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 92.01065246338216
      },
      "perturb_pair": {
       "accuracy": 0.9375,
       "cc": 88.56210813454986,
       "cc_by_type": {
        "choice": 93.993993993994,
        "noul": 100,
        "score": 71.69233040965554
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 88.56210813454986
      },
      "usecase": {
       "accuracy": 0.7818181818181819,
       "cc": 64.20114507147237,
       "cc_by_type": {
        "choice": 74.26470588235296,
        "noul": 45.83333333333333,
        "score": 72.5053959987308
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 64.20114507147237
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.875,
       "cc": 77.53113811828422,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 100,
        "score": 70.68865245009076
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 77.53113811828422
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.8461538461538461,
       "cc": 50.26735599928611,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 17.468734664524987
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 50.26735599928611
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 83.29394929791897,
       "cc_by_type": {
        "choice": 68.14159292035399,
        "noul": 100,
        "score": 81.74025497340291
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 83.29394929791897
      },
      "el": {
       "accuracy": 0.8,
       "cc": 72.18845079261344,
       "cc_by_type": {
        "choice": 83.24022346368716,
        "noul": 33.33333333333333,
        "score": 99.99179558081985
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 72.18845079261344
      },
      "en": {
       "accuracy": 0.8496959165942659,
       "cc": 75.0660808413856,
       "cc_by_type": {
        "choice": 82.86572045740228,
        "noul": 70.50359712230217,
        "score": 71.82892494445237
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 75.0660808413856
      },
      "es": {
       "accuracy": 0.8387096774193549,
       "cc": 73.79728599021023,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 40.96771441435008
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 73.79728599021023
      },
      "fi": {
       "accuracy": 1,
       "cc": 99.97222202004065,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.91666606012194
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 99.97222202004065
      },
      "fr": {
       "accuracy": 0.9285714285714286,
       "cc": 91.69144203830236,
       "cc_by_type": {
        "choice": 78.82352941176471,
        "noul": 100,
        "score": 96.25079670314238
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 91.69144203830236
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 56.7440794132958,
       "cc_by_type": {
        "choice": 99.99999999999999,
        "noul": 0,
        "score": 70.23223823988742
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 56.7440794132958
      },
      "id": {
       "accuracy": 1,
       "cc": 100,
       "cc_by_type": {
        "choice": 100,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 100
      },
      "it": {
       "accuracy": 0.9333333333333333,
       "cc": 78.3981666466817,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 35.19449994004508
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 78.3981666466817
      },
      "ja": {
       "accuracy": 0.64,
       "cc": 36.9676471604448,
       "cc_by_type": {
        "choice": 85.70293884034949,
        "noul": 19.999999999999996,
        "score": 5.200002640984913
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 36.9676471604448
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 17.49673391023856,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": 52.49020173071568
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 17.49673391023856
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.95856075147947,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.87568225443837
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.95856075147947
      },
      "nl": {
       "accuracy": 0.6,
       "cc": -15.46724452134869,
       "cc_by_type": {
        "choice": 100,
        "noul": -100,
        "score": -46.40173356404607
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 0
      },
      "no": {
       "accuracy": 0.5,
       "cc": 17.741935483870968,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 17.741935483870968
      },
      "pl": {
       "accuracy": 0.8717948717948718,
       "cc": 75.54086356569267,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 74.24002616047201
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 75.54086356569267
      },
      "pt": {
       "accuracy": 0.8214285714285714,
       "cc": 79.12997344945892,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 69.21078069015635
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 79.12997344945892
      },
      "sv": {
       "accuracy": 0.7272727272727273,
       "cc": 61.17170090469063,
       "cc_by_type": {
        "choice": 74.46808510638299,
        "noul": 60.00000000000001,
        "score": 49.0470176076889
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 61.17170090469063
      },
      "tr": {
       "accuracy": 0.8461538461538461,
       "cc": 66.2952315244401,
       "cc_by_type": {
        "choice": 100,
        "noul": 50,
        "score": 48.88569457332029
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 66.2952315244401
      },
      "uk": {
       "accuracy": 0.8571428571428571,
       "cc": 88.06941269523666,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 96.46630260183903
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 88.06941269523666
      },
      "zh": {
       "accuracy": 0.8,
       "cc": 47.48723060422734,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 9.128358479348709
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 47.48723060422734
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8693333333333333,
       "cc": 83.12252434987275,
       "cc_by_type": {
        "choice": 83.12252434987275
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 83.12252434987275
      },
      "noul": {
       "accuracy": 0.8293333333333334,
       "cc": 65.86666666666667,
       "cc_by_type": {
        "noul": 65.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 65.86666666666667
      },
      "score": {
       "accuracy": 0.8026666666666666,
       "cc": 69.99615063036917,
       "cc_by_type": {
        "score": 69.99615063036917
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 69.99615063036917
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 22.494329990087607,
       "cc_by_type": {
        "choice": 68,
        "noul": -25,
        "score": 24.482989970262825
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 22.494329990087607
      },
      "code_linting": {
       "accuracy": 1,
       "cc": 96.10834802982788,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 88.32504408948365
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 96.10834802982788
      },
      "customer_support": {
       "accuracy": 0.5714285714285714,
       "cc": 26.49115104816521,
       "cc_by_type": {
        "choice": 34.959349593495936,
        "noul": 50,
        "score": -5.485896449000305
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 26.49115104816521
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": -2.395818046525098,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -70.34534887641739
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 0
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7647058823529411,
       "cc": 40.98936592070195,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 85.286938341816
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 40.98936592070195
      },
      "feature_extraction": {
       "accuracy": 0.8421052631578947,
       "cc": 55.92594796066326,
       "cc_by_type": {
        "choice": 87.42138364779873,
        "noul": 33.33333333333333,
        "score": 47.023126900857726
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 55.92594796066326
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 62.11035211650731,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 42.85714285714286,
        "score": 63.08175662963396
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 62.11035211650731
      },
      "gaming": {
       "accuracy": 0.8636363636363636,
       "cc": 73.43481413884172,
       "cc_by_type": {
        "choice": 89.54703832752612,
        "noul": 50,
        "score": 80.75740408899902
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 73.43481413884172
      },
      "insurance_claims": {
       "accuracy": 0.7333333333333333,
       "cc": 47.33761496863716,
       "cc_by_type": {
        "choice": 79.16666666666667,
        "noul": -19.999999999999996,
        "score": 82.84617823924482
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 47.33761496863716
      },
      "knowledge_graphs": {
       "accuracy": 0.8636363636363636,
       "cc": 75.98050714558066,
       "cc_by_type": {
        "choice": 100,
        "noul": 66.66666666666667,
        "score": 61.27485477007531
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 75.98050714558066
      },
      "lead_generation": {
       "accuracy": 0.8,
       "cc": 67.25946054476482,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 55.55555555555556,
        "score": 57.8507330554831
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 67.25946054476482
      },
      "legal_compliance": {
       "accuracy": 0.4,
       "cc": 13.00854804556269,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": 33.33333333333333,
        "score": 34.72456886787087
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 13.00854804556269
      },
      "llm_guardrails": {
       "accuracy": 0.8148148148148148,
       "cc": 61.743150197630165,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 76.74576537216112
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 61.743150197630165
      },
      "model_routing": {
       "accuracy": 0.8986175115207373,
       "cc": 80.12327163220625,
       "cc_by_type": {
        "choice": 95.32467532467533,
        "noul": 96.96969696969697,
        "score": 48.07544260224647
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 80.12327163220625
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 52.81259558276948,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 0,
        "score": 81.47041630684969
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 52.81259558276948
      },
      "recruiting": {
       "accuracy": 0.8333333333333334,
       "cc": 65.81280362944747,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 80.81816947701466
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 65.81280362944747
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 24.298872150201333,
       "cc_by_type": {
        "choice": 85.22167487684729,
        "noul": -50,
        "score": 37.674941573756705
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 24.298872150201333
      },
      "scientific_discovery": {
       "accuracy": 0.5652173913043478,
       "cc": 24.248186712768014,
       "cc_by_type": {
        "choice": 27.536231884057973,
        "noul": -33.333333333333336,
        "score": 78.5416615875794
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 24.248186712768014
      },
      "search_retrieval": {
       "accuracy": 1,
       "cc": 99.94054161265387,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 99.8216248379616
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 99.94054161265387
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      86.55277391297311,
      91.6227365906473
     ],
     "I": [
      70.21559237844008,
      78.18480469530246
     ],
     "capability": [
      78.79489368820927,
      84.4933851527466
     ],
     "composite": null,
     "seed": 84
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 90.35330442455489,
     "capability": 82.59113932461057,
     "intelligence": 74.82897422466625
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8693333333333333,
      "calibration": 88.60461564741084,
      "cc": 83.12252434987275,
      "cc_by_type": {
       "choice": 83.12252434987275
      },
      "cc_tiered": 84.79263141868302,
      "cc_tiered_open": 87.81849238331058,
      "cc_tiered_sealed": 81.76677045405546,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 83.12252434987275
     },
     "noul": {
      "accuracy": 0.8293333333333334,
      "calibration": 87.76065015691402,
      "cc": 65.86666666666667,
      "cc_by_type": {
       "noul": 65.86666666666667
      },
      "cc_tiered": 69.44608087703705,
      "cc_tiered_open": 80.07729468599035,
      "cc_tiered_sealed": 58.81486706808374,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 65.86666666666667
     },
     "score": {
      "accuracy": 0.8026666666666666,
      "calibration": 94.6946474693398,
      "cc": 69.99615063036917,
      "cc_by_type": {
       "score": 69.99615063036917
      },
      "cc_tiered": 70.24821037827869,
      "cc_tiered_open": 71.9460054472965,
      "cc_tiered_sealed": 68.55041530926088,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 69.99615063036917
     }
    },
    "raw_on_A": {
     "calibration": 89.05513895349148,
     "capability": 81.74574955505761,
     "intelligence": 74.43636015662376
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.5790528116713,
    "Emphasis on Accuracy 60:20:20": 72.21974476501191,
    "Emphasis on Cost 20:20:60": 64.56909273321604,
    "Emphasis on Speed 20:60:20": 75.92083661463992,
    "Intelligence only": 74.82897422466625
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "org2ai (Hugging Face)",
   "axes": {
    "calibration": 88.05577650422971,
    "cost": 70.95910393447916,
    "intelligence": 65.50854607947286,
    "speed": 92.01872463451872
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.0364505263232707,
      "ece_score": 92.70989473534586,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 315,
      "n_tvd": 0,
      "score": 92.70989473534586,
      "score_before_time_limit": 92.70989473534586,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.12762808226321673,
      "ece": 0.09810736388535705,
      "n": 375,
      "score": 80.37852722292858,
      "score_before_time_limit": 80.37852722292858,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9813333333333333,
      "interval90_width": 0.44444444444444464,
      "n": 375,
      "nrps": 0.07497524415840688,
      "score": 91.07890755441471,
      "score_before_time_limit": 91.07890755441471,
      "time_limit_failed_share": 0,
      "top_ece": 0.05172330237664942
     }
    },
    "score": 88.05577650422971
   },
   "capability": 76.78216129185128,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 790,996 input + 0 output tokens over 1,485 decisions = 532.657239057 tokens/decision. GPU-time cost USD 0.0092903027917012/1,000; old list-price cost USD 0.0479391515151515/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0092903027917012/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.11893433999999999
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009290302791701198
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Wald 26B-A4B (Gemma 4 26B-A4B)",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-23",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), pay-as-you-go non-spot, instance type 1RTXPRO6000.30V at USD 2.549/h. Stock vLLM 0.30.0 (127.0.0.1:8371) and the author's reader eval.systemone_vllm (127.0.0.1:8000) run in ONE docker container with network none; the reader was timed over loopback, one request at a time. Network cut proven from inside the container before any item hash or model load (loopback only, no default route, DNS and TCP fail); read-only code, read-only model and input, empty credential variables (names-only proof), HF offline flags. Measured 23 Oct 2026 (VM 06:50-07:07Z; measurement 338 s, total 981 s; scored run 204 s) on the frozen v1.6-regular-20261012 self-hosted input (sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, re-checked inside the container). The input was shredded from the node and its absence verified, and the VM was removed by exact id afterwards (teardown confirmed 07:07Z).",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 71.5000217659513,
    "I_sealed": 59.51707039299441,
    "base": 65.50854607947286,
    "excess": 2.143424602601179,
    "gap": 11.98295137295689,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 73.59023844561248,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396664,
       "judge": 72.31377719182598,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 83.56521739130434,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 73.91304347826086,
       "judge": 80,
       "standard": 100
      }
     },
     "open|score": {
      "cc": 57.34460946093709,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.66872201243804,
       "hard": 55.94711638549248,
       "judge": 62.13648005986813,
       "standard": 54.7897334376793
      }
     },
     "sealed|choice": {
      "cc": 75.24186878564784,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 86.6581956797967,
       "hard": 69.81612119810339,
       "judge": 73.55533790401567,
       "standard": 82.91499683611053
      }
     },
     "sealed|noul": {
      "cc": 56.17462002742497,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 82.75862068965519,
       "hard": 47.272727272727266,
       "judge": 53.846153846153854,
       "standard": 64.17910447761194
      }
     },
     "sealed|score": {
      "cc": 47.134722365910434,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 65.87205357327875,
       "hard": 48.67115104064259,
       "judge": 37.1986896099809,
       "standard": 49.597248546656246
      }
     }
    }
   },
   "jevbench_score": 77.54703220970119,
   "key": "wald-26b-a4b",
   "last_measured_on": "2026-10-23",
   "licence": "Apache-2.0 (model card and Hub metadata; base google/gemma-4-26B-A4B-it: apache-2.0 in its card metadata with a link to the Gemma 4 licence), public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "org2ai/Wald-26B-A4B@4a5d0b94bff26a8bba359ef311b3e535cefae901",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.824,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 6,
   "ranked": true,
   "ranks": {
    "A": 6,
    "B": 14,
    "C": 5,
    "capability": 56
   },
   "repo": "https://huggingface.co/org2ai/Wald-26B-A4B",
   "scores": {
    "A": 77.54703220970119,
    "B": 74.79791410109989,
    "C": 77.54703220970119
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.2362664810000524,
    "p50_s_raw": 0.043133240500026204,
    "p95_s_adjusted": 0.26590433870004515,
    "p95_s_raw": 0.057952169350022585
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHORS' OWN READER, run unmodified, in front of stock vLLM. Author code: 10 Python modules of org2ai/Wald-26B-A4B at 4a5d0b94bff26a8bba359ef311b3e535cefae901 (eval/__init__.py, eval/systemone_vllm.py, eval/vllm_letter.py, kev/__init__.py, kev/api.py, midtrain/__init__.py, midtrain/encode.py, midtrain/letter.py, midtrain/rawprompt.py, midtrain/tokens.py; CODE_MANIFEST.json sha256 5ce8feb796d52add3dfa4520fa0fe418c4eaaf4551591ad7d2485aca72aefaa8), hash-verified, mounted read-only at /code. The unshipped modules midtrain/chat_lora.py (contains snapshot_download and torch.load(weights_only=False)) and midtrain/gemma_moe.py are NOT shipped and fail closed with ImportError if reached; the author's run.sh (which installs packages from the network) was NEVER run. Instead we reproduce its settings (EFFORT=none, checkpoint 05901-c33): vLLM 0.30.0 --served-model-name wald-26b-a4b --max-model-len 131072 --gpu-memory-utilization 0.90 --max-num-seqs 128 --max-logprobs 64 --seed 0 --language-model-only, and the author's reader python3 -m eval.systemone_vllm --vllm http://127.0.0.1:8371 --gate 0 --budget 512 --wide knockout --template chat --prompt-format plain --max-model-len 131072 --noul-decisive-floor 0.81 --host 127.0.0.1 --port 8000. Image: official vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90 plus two hash-pinned wheels (transformers 5.17.0, huggingface-hub 1.33.0) built offline on the VM (image sha256:dc93fedeea119e31b524c2352f397326d3b315cf129d33c0a7001cfce7c8e56e); torch 2.13.0+cu130, tokenizers 0.23.2. DISCLOSED DEVIATIONS from run.sh: run.sh not executed; reader bound to 127.0.0.1 (run.sh uses 0.0.0.0); DO_NOT_TRACK and HF_HUB_DISABLE_TELEMETRY added; the source-pins check is replaced by sha256 verification of all 10 files before upload. The submitted code was reviewed before the run, with no blocking findings (no download, subprocess, env read, exec/eval, pickle, torch.load or telemetry in the 10 shipped modules). The source and runner were reviewed; findings concerning the model card and measurement-window guard were resolved in the preregistration amendments before the run. Weights: org2ai/Wald-26B-A4B at 4a5d0b94bff26a8bba359ef311b3e535cefae901 (Apache-2.0, public, not gated, 39 files, 11 safetensors shards totalling 51,612,008,604 bytes (51,644,629,065 bytes for all 39 files), no pickle; Gemma-4-26B-A4B-it after full-parameter decision training, base google/gemma-4-26B-A4B-it@4d7ae4984b7db7de8f8457170b3f1a419ee76d52), sha256-verified file by file against the Hub LFS oids (MODEL_MANIFEST.json sha256 cfbc591cc731191f99f0585f334a2735b5e984deb6b7808cf9a8357d89dcbe5e). FIRST MEASUREMENT (found by our own intake; Author submission, a submission by the authors; same family as the published wald-4b-v2-1 row, a different base). AUTHOR'S OWN READER, on our hardware, not the author's run.sh (see code_revision). BENCHMARK-AWARE OUTPUT POLICY (author-shipped, measured as shipped): the reader runs with --noul-decisive-floor 0.81 (eval/systemone_vllm.py:36-42, 488-502, and serving.json). It moves a Noul probability inside (0.19, 0.81) to the nearer edge; the author's own comment says 'JevBench v1.5/v1.6 (O1S) scores 1-f<P<f as an abstention'. It is a pure function of the model's own probability, with no item ids or item strings. EFFECT ON THIS RUN: 123 of 375 Noul answers (32.8 %) sit exactly on 0.81 or 0.19, no answer lies inside (0.19, 0.81), so the abstention rate is 0.0; the Noul calibration is measured as shipped and is not comparable to an unfloored run (the unfloored probabilities are not recorded). This is a disclosure per exposure ruling on file. AUTHOR EXPOSURE (self-reported, not verified): the README claims 219/231 on the JevBench public set (Jev 1.13: 200/231), run by the authors; training on public items is possible and was not checked by us (no overlap check; no JevBench use or denial declared). 10 s RULE: max measured decision latency 9.79 s (median 0.043 s, p95 0.058 s, 99th percentile 8.63 s); 0 decisions over 10 s, but the slowest decisions are close to the limit (in the smoke a 78.5k-token request took 8.8 s cold and 0.18 s warm). DETERMINISM: first 50 positions run twice, only 1/50 bit-identical, 0 argmax flips, max probability difference 0.042 (disclosed; the cause was not investigated). USAGE OFFSET: server usage.input_tokens = our offline count + 1 on every item; see cost_basis. CONTEXT: the author's limit is 131,072 tokens per read and a longer read is refused with HTTP 422, never truncated; the longest sealed item is 81,725 tokens, so nothing was refused; the smoke accepted 131,070 tokens. The scored configuration is EFFORT=none (no thinking); no sealed item had more than 26 options, so the knockout mode was never used. PRICING: the frozen module has no explicit listing decision for the card's exact base spelling; the estimate follows the published sibling rows (see cost_basis). No outside request for this model.",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.8636363636363636,
       "cc": 62.28668771673929,
       "cc_by_type": {
        "choice": 83.33333333333333,
        "noul": 100,
        "score": 3.526729816884533
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 62.28668771673929
      },
      "calibration_gen": {
       "accuracy": 0.7128712871287128,
       "cc": 61.329080094640695,
       "cc_by_type": {
        "choice": 71.36797454931072,
        "noul": 60.00000000000001,
        "score": 52.61926573461137
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 61.329080094640695
      },
      "composite_gen": {
       "accuracy": 0.8910891089108911,
       "cc": 79.88818031266844,
       "cc_by_type": {
        "choice": 90.2439024390244,
        "noul": 85.18518518518519,
        "score": 64.23545331379572
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 79.88818031266844
      },
      "fairness": {
       "accuracy": 0.875,
       "cc": 80.95766193104647,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 82.87298579313939
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 80.95766193104647
      },
      "jaggedness": {
       "accuracy": 0.6888888888888889,
       "cc": 49.006962191036905,
       "cc_by_type": {
        "choice": 72.56977863330125,
        "noul": 40.54054054054055,
        "score": 33.910567399268906
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 49.006962191036905
      },
      "llm_router": {
       "accuracy": 0.8242811501597445,
       "cc": 71.5864116735259,
       "cc_by_type": {
        "choice": 73.84872765695167,
        "noul": 83.50515463917525,
        "score": 57.40535272445078
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 71.5864116735259
      },
      "long_state": {
       "accuracy": 0.6,
       "cc": -7.025166452017032,
       "cc_by_type": {
        "choice": 61.29032258064516,
        "noul": -100,
        "score": 17.63417806330375
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.6296296296296297,
       "cc": 40.61069467435372,
       "cc_by_type": {
        "choice": 50.51546391752578,
        "noul": 75,
        "score": -3.6833798944646334
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 40.61069467435372
      },
      "multilingual": {
       "accuracy": 0.7527272727272727,
       "cc": 61.11914207752587,
       "cc_by_type": {
        "choice": 73.58982719269645,
        "noul": 58.33333333333333,
        "score": 51.43426570654784
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 61.11914207752587
      },
      "negation_pair_gen": {
       "accuracy": 0.8255813953488372,
       "cc": 59.75609874358286,
       "cc_by_type": {
        "choice": 93.33333333333333,
        "noul": 53.33333333333334,
        "score": 32.60162956408189
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 59.75609874358286
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 80.35665685016336,
       "cc_by_type": {
        "choice": 72.97297297297297,
        "noul": 100,
        "score": 68.0969975775171
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 80.35665685016336
      },
      "usecase": {
       "accuracy": 0.7,
       "cc": 54.9190856886155,
       "cc_by_type": {
        "choice": 68.1372549019608,
        "noul": 50,
        "score": 46.62000216388572
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 54.9190856886155
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 29.362565407181737,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": 100,
        "score": -54.76944663559764
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 29.362565407181737
      },
      "cs": {
       "accuracy": 0.5714285714285714,
       "cc": 68.42105263157895,
       "cc_by_type": {
        "choice": 36.84210526315789,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 68.42105263157895
      },
      "da": {
       "accuracy": 0.6153846153846154,
       "cc": 16.94109176007718,
       "cc_by_type": {
        "choice": 79.26267281105991,
        "noul": -33.333333333333336,
        "score": 4.893935802504967
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 16.94109176007718
      },
      "de": {
       "accuracy": 0.8125,
       "cc": 72.45376024931926,
       "cc_by_type": {
        "choice": 76.10619469026548,
        "noul": 75,
        "score": 66.25508605769228
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 72.45376024931926
      },
      "el": {
       "accuracy": 0.8,
       "cc": 77.55632535362075,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 66.66666666666667,
        "score": 99.52186246682126
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 77.55632535362075
      },
      "en": {
       "accuracy": 0.7819287576020851,
       "cc": 64.4227407367809,
       "cc_by_type": {
        "choice": 77.15429394320303,
        "noul": 64.02877697841727,
        "score": 52.0851512887224
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 64.4227407367809
      },
      "es": {
       "accuracy": 0.8064516129032258,
       "cc": 65.13915722074792,
       "cc_by_type": {
        "choice": 90.2120717781403,
        "noul": 100,
        "score": 5.205399884103468
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 65.13915722074792
      },
      "fi": {
       "accuracy": 0.625,
       "cc": 55.65429624620543,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 33.33333333333333,
        "score": 99.20332589708623
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 55.65429624620543
      },
      "fr": {
       "accuracy": 0.7142857142857143,
       "cc": 74.9477771440457,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 100,
        "score": 67.19627260860767
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 74.9477771440457
      },
      "hi": {
       "accuracy": 0.8095238095238095,
       "cc": 57.73566501094746,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 75,
        "score": 31.726548105468066
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 57.73566501094746
      },
      "id": {
       "accuracy": 0.75,
       "cc": 0,
       "cc_by_type": {
        "choice": 100,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 0
      },
      "it": {
       "accuracy": 0.8,
       "cc": 60.337669469601714,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": -4.957373508575991
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 60.337669469601714
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 36.332437701064066,
       "cc_by_type": {
        "choice": 57.108816521048446,
        "noul": 39.99999999999999,
        "score": 11.888496582143748
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 36.332437701064066
      },
      "ko": {
       "accuracy": 0.6,
       "cc": 59.15338662842026,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100,
        "score": 21.904604329705222
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 59.15338662842026
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.60356597056011,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.81069791168032
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.60356597056011
      },
      "nl": {
       "accuracy": 0.6,
       "cc": 15.363590673359726,
       "cc_by_type": {
        "choice": 74.02597402597404,
        "noul": -33.333333333333336,
        "score": 5.398131327438471
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 15.363590673359726
      },
      "no": {
       "accuracy": 0.6666666666666666,
       "cc": 42.74193548387097,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 42.74193548387097
      },
      "pl": {
       "accuracy": 0.8205128205128205,
       "cc": 73.71413499154939,
       "cc_by_type": {
        "choice": 92.382564536606,
        "noul": 60.00000000000001,
        "score": 68.75984043804215
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 73.71413499154939
      },
      "pt": {
       "accuracy": 0.7142857142857143,
       "cc": 67.58621040803877,
       "cc_by_type": {
        "choice": 46.96523276370065,
        "noul": 100,
        "score": 55.79339846041565
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 67.58621040803877
      },
      "sv": {
       "accuracy": 0.6363636363636364,
       "cc": 56.70325456902729,
       "cc_by_type": {
        "choice": 48.93617021276595,
        "noul": 60.00000000000001,
        "score": 61.173593494315924
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 56.70325456902729
      },
      "tr": {
       "accuracy": 0.6923076923076923,
       "cc": 51.526643189721035,
       "cc_by_type": {
        "choice": 78.26086956521739,
        "noul": 100,
        "score": -23.680939996054274
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 51.526643189721035
      },
      "uk": {
       "accuracy": 1,
       "cc": 99.06371815169098,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 97.19115445507296
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 99.06371815169098
      },
      "zh": {
       "accuracy": 1,
       "cc": 76.19757397633792,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 28.592721929013752
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 76.19757397633792
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.8146666666666667,
       "cc": 76.06153963910522,
       "cc_by_type": {
        "choice": 76.06153963910522
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 76.06153963910522
      },
      "noul": {
       "accuracy": 0.824,
       "cc": 64.8,
       "cc_by_type": {
        "noul": 64.8
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 64.8
      },
      "score": {
       "accuracy": 0.6453333333333333,
       "cc": 50.36096329833563,
       "cc_by_type": {
        "score": 50.36096329833563
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 50.36096329833563
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 18.962327091129648,
       "cc_by_type": {
        "choice": 20,
        "noul": 50,
        "score": -13.11301872661106
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 18.962327091129648
      },
      "code_linting": {
       "accuracy": 0.7916666666666666,
       "cc": 71.39826677963767,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 14.194800338912984
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 71.39826677963767
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 44.39453904633217,
       "cc_by_type": {
        "choice": 67.47967479674797,
        "noul": 50,
        "score": 15.703942342248522
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 44.39453904633217
      },
      "demand_forecasting": {
       "accuracy": 0.47058823529411764,
       "cc": 11.335982228981843,
       "cc_by_type": {
        "choice": 29.82456140350877,
        "noul": 33.33333333333333,
        "score": -29.14994804989657
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 11.335982228981843
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 46.06728146848554,
       "cc_by_type": {
        "choice": 85.50724637681158,
        "noul": 33.33333333333333,
        "score": 19.361264695311707
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 46.06728146848554
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": -25.58952390664666,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 66.66666666666667,
        "score": -218.27800568220414
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.7666666666666667,
       "cc": 50.44808275364195,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 14.28571428571428,
        "score": 56.666377112466456
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 50.44808275364195
      },
      "gaming": {
       "accuracy": 0.6363636363636364,
       "cc": 51.727112859171505,
       "cc_by_type": {
        "choice": 58.18815331010453,
        "noul": 50,
        "score": 46.993185267409984
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 51.727112859171505
      },
      "insurance_claims": {
       "accuracy": 0.4,
       "cc": 30.020847503119555,
       "cc_by_type": {
        "choice": 37.5,
        "noul": 19.999999999999996,
        "score": 32.562542509358664
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 30.020847503119555
      },
      "knowledge_graphs": {
       "accuracy": 0.5454545454545454,
       "cc": 40.73034293084367,
       "cc_by_type": {
        "choice": 48.608137044967876,
        "noul": 33.33333333333333,
        "score": 40.2495584142298
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 40.73034293084367
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 57.44797224134985,
       "cc_by_type": {
        "choice": 88.3720930232558,
        "noul": 33.33333333333333,
        "score": 50.638490367460435
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 57.44797224134985
      },
      "legal_compliance": {
       "accuracy": 0.6,
       "cc": 42.85782288237548,
       "cc_by_type": {
        "choice": 3.2258064516129026,
        "noul": 100,
        "score": 25.347662195513553
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 42.85782288237548
      },
      "llm_guardrails": {
       "accuracy": 0.7777777777777778,
       "cc": 57.469447177637996,
       "cc_by_type": {
        "choice": 88.48368522072937,
        "noul": 19.999999999999996,
        "score": 63.924656312184624
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 57.469447177637996
      },
      "model_routing": {
       "accuracy": 0.8525345622119815,
       "cc": 74.45948584990684,
       "cc_by_type": {
        "choice": 80.12987012987014,
        "noul": 96.96969696969697,
        "score": 46.2788904501534
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 74.45948584990684
      },
      "moderation_trust_safety": {
       "accuracy": 0.8333333333333334,
       "cc": 82.11215077126832,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100,
        "score": 69.3690818723462
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 82.11215077126832
      },
      "recruiting": {
       "accuracy": 0.7222222222222222,
       "cc": 57.71735862375096,
       "cc_by_type": {
        "choice": 83.28690807799443,
        "noul": 33.33333333333333,
        "score": 56.531834459925136
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 57.71735862375096
      },
      "risk_assessment": {
       "accuracy": 0.6111111111111112,
       "cc": 53.55195584334256,
       "cc_by_type": {
        "choice": 40.88669950738917,
        "noul": 100,
        "score": 19.769168022638517
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 53.55195584334256
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 33.766528904412525,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 0,
        "score": 44.777847582802785
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 33.766528904412525
      },
      "search_retrieval": {
       "accuracy": 0.8333333333333334,
       "cc": 86.18780733194313,
       "cc_by_type": {
        "choice": 78.64768683274022,
        "noul": 100,
        "score": 79.91573516308917
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 86.18780733194313
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.6478419735516,
      88.60393644006602
     ],
     "I": [
      60.90990743631795,
      69.0211072229648
     ],
     "capability": [
      72.50042821110986,
      78.13972271554701
     ],
     "composite": null,
     "seed": 86
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 88.05577650422971,
     "capability": 76.78216129185128,
     "intelligence": 65.50854607947286
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8146666666666667,
      "calibration": 92.70989473534586,
      "cc": 76.06153963910522,
      "cc_by_type": {
       "choice": 76.06153963910522
      },
      "cc_tiered": 74.41605361563016,
      "cc_tiered_open": 73.59023844561248,
      "cc_tiered_sealed": 75.24186878564784,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.06153963910522
     },
     "noul": {
      "accuracy": 0.824,
      "calibration": 80.37852722292858,
      "cc": 64.8,
      "cc_by_type": {
       "noul": 64.8
      },
      "cc_tiered": 69.86991870936465,
      "cc_tiered_open": 83.56521739130434,
      "cc_tiered_sealed": 56.17462002742497,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 64.8
     },
     "score": {
      "accuracy": 0.6453333333333333,
      "calibration": 91.07890755441471,
      "cc": 50.36096329833563,
      "cc_by_type": {
       "score": 50.36096329833563
      },
      "cc_tiered": 52.239665913423764,
      "cc_tiered_open": 57.34460946093709,
      "cc_tiered_sealed": 47.134722365910434,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 50.36096329833563
     }
    },
    "raw_on_A": {
     "calibration": 84.82817901188946,
     "capability": 73.65390073325852,
     "intelligence": 62.47962245462759
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 74.58018503952037,
    "Emphasis on Accuracy 60:20:20": 70.66585529533967,
    "Emphasis on Cost 20:20:60": 73.08829117116471,
    "Emphasis on Speed 20:60:20": 80.69739533257618,
    "Intelligence only": 65.50854607947286
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": null,
   "axes": {
    "calibration": 78.10292780844634,
    "cost": 70.53846200925558,
    "intelligence": 41.00304543028662,
    "speed": 82.31428682922845
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12842390280324772,
      "ece_score": 74.31521943935046,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 322,
      "n_tvd": 0,
      "score": 73.62161072458319,
      "score_before_time_limit": 74.31521943935046,
      "time_limit_failed_share": 0.009333333333333334
     },
     "noul": {
      "abstention_rate": 0.33333333333333337,
      "brier": 0.11388917204323451,
      "ece": 0.06372845741047697,
      "n": 375,
      "score": 87.2543085179046,
      "score_before_time_limit": 87.2543085179046,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.8053333333333333,
      "interval90_width": 0.24711111111111111,
      "n": 375,
      "nrps": 0.10762527231724621,
      "score": 73.43286418285125,
      "score_before_time_limit": 73.6292087394899,
      "time_limit_failed_share": 0.0026666666666666666,
      "top_ece": 0.20989527644647793
     }
    },
    "score": 78.10292780844634
   },
   "capability": 59.55298661936648,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 1,394,108 input + 50,098 output tokens over 1,479 decisions = 976.47464503 tokens/decision. GPU-time cost USD 0.00959513890794073/1,000; old list-price cost USD 0.033358985801217/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00959513890794073/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1479,
     "rule": "common",
     "usd_per_1000_all_items": 0.12006908000000001
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.009595138907940727
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 8,
    "n_over_limit_answered_ok": 8,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Wald 4B",
   "draw": {
    "drawn_on": "2026-10-09",
    "g_med": 7.088097744249485,
    "id": "v1.6-fastlane-20261009",
    "label": "Fast-lane draw",
    "measured_on": "2026-10-09",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.4",
    "results_sha256": "f3d57e0705da2c565bc0925a03a0fe6aa351d3cff37e71816a92e6d27cc3e7d7",
    "status": "anchor-runs-pending"
   },
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 42.50880508565272,
    "I_sealed": 39.49728577492053,
    "base": 41.00304543028662,
    "excess": -4.0765784335172945,
    "gap": 3.01151931073219,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.2690514819802,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 66.02705253224286,
       "judge": 80.22412656558998,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 16.316770186335404,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 71.42857142857142,
       "hard": 30.434782608695656,
       "judge": -9.999999999999998,
       "standard": 0
      }
     },
     "open|score": {
      "cc": 40.94059358864256,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 53.042005572750426,
       "hard": 44.3073169155134,
       "judge": 46.64410687719149,
       "standard": 19.601171010023556
      }
     },
     "sealed|choice": {
      "cc": 63.21007145303903,
      "n": {
       "easy": 73,
       "hard": 254,
       "judge": 141,
       "standard": 132
      },
      "tiers": {
       "easy": 83.3316184792674,
       "hard": 56.443444006752955,
       "judge": 60.82004555808657,
       "standard": 70.26759167492567
      }
     },
     "sealed|noul": {
      "cc": 18.474093177291948,
      "n": {
       "easy": 48,
       "hard": 116,
       "judge": 67,
       "standard": 69
      },
      "tiers": {
       "easy": 54.16666666666667,
       "hard": 15.517241379310342,
       "judge": 4.477611940298498,
       "standard": 27.536231884057962
      }
     },
     "sealed|score": {
      "cc": 36.8076926944306,
      "n": {
       "easy": 36,
       "hard": 111,
       "judge": 74,
       "standard": 79
      },
      "tiers": {
       "easy": 58.68738567055003,
       "hard": 33.084604932395,
       "judge": 27.44752759181396,
       "standard": 47.35426938436701
      }
     }
    }
   },
   "jevbench_score": 42.35067865852022,
   "key": "wald-4b-v2",
   "last_measured_on": "2026-10-09",
   "listing": "ranked",
   "measured_in": "v1.6.2",
   "measurement_date_status": "actual native measurement completion day (UTC)",
   "method": "v1.7",
   "model_pin": "919a9d72a65379271005ed0945bfc72742dad3af",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 78.39957889891498,
     "cost": 54.30361484501323,
     "intelligence": 41.11158964477434,
     "speed": 82.20577865115169
    },
    "board_release": "v1.6.43",
    "capability": 59.75558427184466,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 61,
    "capability_rank": 86,
    "composite_ci95": {
     "A": [
      27.711939304816244,
      52.55504034148649
     ],
     "B": [
      24.89942588247082,
      49.38746598254962
     ],
     "C": [
      19.244402295011277,
      36.49655579269895
     ]
    },
    "jevbench_score": 39.9690949735785,
    "measured_in": "v1.6.2",
    "method": "v1.6",
    "open_board_rank": 54,
    "rank": 83,
    "ranks": {
     "A": 83,
     "B": 84,
     "C": 85,
     "capability": 133
    },
    "scores": {
     "A": 39.9690949735785,
     "B": 36.74952891978881,
     "C": 27.756315953873955
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.21244139187037944,
     "p50_s_raw": 0.031220695935189724,
     "p95_s_adjusted": 2.8325852981768525,
     "p95_s_raw": 1.3412926490884263
    },
    "views": {
     "Balanced 33:33:33": 36.94101579201056,
     "Emphasis on Accuracy 60:20:20": 32.64384498780741,
     "Emphasis on Cost 20:20:60": 36.849381766766875,
     "Emphasis on Speed 20:60:20": 42.663206822382165,
     "Intelligence only": 27.7941118378976
    }
   },
   "rank": 143,
   "ranked": true,
   "ranks": {
    "A": 143,
    "B": 145,
    "C": 148,
    "capability": 168
   },
   "repo": "https://huggingface.co/org2ai/Wald-4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 42.35067865852022,
    "B": 38.25120899749685,
    "C": 29.410193512861273
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23851143810898065,
    "p50_s_raw": 0.04425571905449033,
    "p95_s_adjusted": 2.460719473008066,
    "p95_s_raw": 1.1553597365040331
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.72,
       "cc": 68.2119055157796,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 50,
        "score": 99.08016099178325
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 2
       },
       "score": 68.2119055157796
      },
      "calibration_gen": {
       "accuracy": 0.7395833333333334,
       "cc": 43.20138852122932,
       "cc_by_type": {
        "choice": 75.30864197530865,
        "noul": 25,
        "score": 29.295523588379304
       },
       "low_n": false,
       "n": 96,
       "n_by_type": {
        "choice": 32,
        "noul": 24,
        "score": 40
       },
       "score": 43.20138852122932
      },
      "composite_gen": {
       "accuracy": 0.61,
       "cc": 27.594758393773663,
       "cc_by_type": {
        "choice": 73.98373983739837,
        "noul": -12,
        "score": 20.80053534392261
       },
       "low_n": false,
       "n": 100,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 34
       },
       "score": 27.594758393773663
      },
      "fairness": {
       "accuracy": 0.8333333333333334,
       "cc": 81.41361209300184,
       "cc_by_type": {
        "choice": 79.66101694915254,
        "noul": 100,
        "score": 64.579819329853
       },
       "low_n": false,
       "n": 42,
       "n_by_type": {
        "choice": 21,
        "noul": 6,
        "score": 15
       },
       "score": 81.41361209300184
      },
      "jaggedness": {
       "accuracy": 0.5493827160493827,
       "cc": 8.823790192360185,
       "cc_by_type": {
        "choice": 53.94899727655361,
        "noul": -42.85714285714286,
        "score": 15.379516157669803
       },
       "low_n": false,
       "n": 162,
       "n_by_type": {
        "choice": 91,
        "noul": 28,
        "score": 43
       },
       "score": 8.823790192360185
      },
      "llm_router": {
       "accuracy": 0.775974025974026,
       "cc": 64.44463445590775,
       "cc_by_type": {
        "choice": 69.49968786230271,
        "noul": 70.83333333333333,
        "score": 53.00088217208718
       },
       "low_n": false,
       "n": 308,
       "n_by_type": {
        "choice": 151,
        "noul": 96,
        "score": 61
       },
       "score": 64.44463445590775
      },
      "long_state": {
       "accuracy": 0.2857142857142857,
       "cc": -27.03945295275031,
       "cc_by_type": {
        "choice": 13.793103448275854,
        "noul": 0,
        "score": -94.91146230652679
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 15,
        "noul": 2,
        "score": 4
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.5,
       "cc": 7.403242931791567,
       "cc_by_type": {
        "choice": 47.36842105263158,
        "noul": -75,
        "score": 49.841307742743126
       },
       "low_n": false,
       "n": 34,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 13
       },
       "score": 7.403242931791567
      },
      "multilingual": {
       "accuracy": 0.6491228070175439,
       "cc": 38.093835945068015,
       "cc_by_type": {
        "choice": 63.57409713574097,
        "noul": 9.756097560975618,
        "score": 40.95131313848746
       },
       "low_n": false,
       "n": 285,
       "n_by_type": {
        "choice": 140,
        "noul": 82,
        "score": 63
       },
       "score": 38.093835945068015
      },
      "negation_pair_gen": {
       "accuracy": 0.5204081632653061,
       "cc": 20.521786356237072,
       "cc_by_type": {
        "choice": 59.99999999999999,
        "noul": -29.411764705882348,
        "score": 30.977123774593572
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 40,
        "noul": 34,
        "score": 24
       },
       "score": 20.521786356237072
      },
      "option_order": {
       "accuracy": 0.873015873015873,
       "cc": 83.82204246713852,
       "cc_by_type": {
        "choice": 83.82204246713852
       },
       "low_n": false,
       "n": 63,
       "n_by_type": {
        "choice": 63
       },
       "score": 83.82204246713852
      },
      "perturb_pair": {
       "accuracy": 0.8064516129032258,
       "cc": 72.61751779733824,
       "cc_by_type": {
        "choice": 71.69811320754718,
        "noul": 83.33333333333333,
        "score": 62.821106851134225
       },
       "low_n": false,
       "n": 62,
       "n_by_type": {
        "choice": 42,
        "noul": 12,
        "score": 8
       },
       "score": 72.61751779733824
      },
      "usecase": {
       "accuracy": 0.6274509803921569,
       "cc": 42.32708851945942,
       "cc_by_type": {
        "choice": 61.16504854368934,
        "noul": 19.999999999999996,
        "score": 45.81621701468893
       },
       "low_n": false,
       "n": 204,
       "n_by_type": {
        "choice": 86,
        "noul": 50,
        "score": 68
       },
       "score": 42.32708851945942
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.47058823529411764,
       "cc": 14.120375284224684,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 0,
        "score": 18.55160204315024
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 4
       },
       "score": 14.120375284224684
      },
      "cs": {
       "accuracy": 0.6666666666666666,
       "cc": 68.9655172413793,
       "cc_by_type": {
        "choice": 37.93103448275862,
        "noul": 100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 68.9655172413793
      },
      "da": {
       "accuracy": 0.6875,
       "cc": 50.49501668418102,
       "cc_by_type": {
        "choice": 68.88504753673294,
        "noul": 50,
        "score": 32.60000251581009
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 4
       },
       "score": 50.49501668418102
      },
      "de": {
       "accuracy": 0.725,
       "cc": 54.187380432772635,
       "cc_by_type": {
        "choice": 70.3581720872787,
        "noul": 27.27272727272727,
        "score": 64.93124193831193
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 17,
        "noul": 11,
        "score": 12
       },
       "score": 54.187380432772635
      },
      "el": {
       "accuracy": 0.47058823529411764,
       "cc": 15.393732253336765,
       "cc_by_type": {
        "choice": 42.857142857142854,
        "noul": -60,
        "score": 63.32405390286744
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 5
       },
       "score": 15.393732253336765
      },
      "en": {
       "accuracy": 0.6757709251101321,
       "cc": 43.45231827001086,
       "cc_by_type": {
        "choice": 65.82108816521045,
        "noul": 24.626865671641784,
        "score": 39.90900097318036
       },
       "low_n": false,
       "n": 1135,
       "n_by_type": {
        "choice": 580,
        "noul": 268,
        "score": 287
       },
       "score": 43.45231827001086
      },
      "es": {
       "accuracy": 0.7333333333333333,
       "cc": 51.07663096036739,
       "cc_by_type": {
        "choice": 75.9197324414716,
        "noul": 27.27272727272727,
        "score": 50.03743316690328
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 8
       },
       "score": 51.07663096036739
      },
      "fi": {
       "accuracy": 0.42857142857142855,
       "cc": 1.9662469414521173,
       "cc_by_type": {
        "choice": 48.71794871794871,
        "noul": -100,
        "score": 57.180792106407644
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 1
       },
       "score": 1.9662469414521173
      },
      "fr": {
       "accuracy": 0.8125,
       "cc": 47.283240126339756,
       "cc_by_type": {
        "choice": 85.5421686746988,
        "noul": -33.333333333333336,
        "score": 89.64088503765379
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 4
       },
       "score": 47.283240126339756
      },
      "hi": {
       "accuracy": 0.6153846153846154,
       "cc": 26.24706077168248,
       "cc_by_type": {
        "choice": 77.818853974122,
        "noul": -33.333333333333336,
        "score": 34.255661674258775
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 8
       },
       "score": 26.24706077168248
      },
      "id": {
       "accuracy": 0.5,
       "cc": 34.42622950819673,
       "cc_by_type": {
        "choice": 34.42622950819673
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 4
       },
       "score": 34.42622950819673
      },
      "it": {
       "accuracy": 0.9230769230769231,
       "cc": 76.36299722208126,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 95.75565833291049
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 2
       },
       "score": 76.36299722208126
      },
      "ja": {
       "accuracy": 0.5,
       "cc": -10.117201418040365,
       "cc_by_type": {
        "choice": 43.172849250197324,
        "noul": 25,
        "score": -98.52445350431842
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 8,
        "score": 5
       },
       "score": 0
      },
      "ko": {
       "accuracy": 0.6666666666666666,
       "cc": 74.45318709325632,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 100,
        "score": 50.02622794643561
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 5,
        "noul": 1,
        "score": 3
       },
       "score": 74.45318709325632
      },
      "mixed": {
       "accuracy": 1,
       "cc": 95.32198614555834,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 85.96595843667502
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 95.32198614555834
      },
      "nl": {
       "accuracy": 0.7058823529411765,
       "cc": 8.120626845785097,
       "cc_by_type": {
        "choice": 71.24600638977637,
        "noul": 0,
        "score": -46.88412585242108
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 4
       },
       "score": 8.120626845785097
      },
      "no": {
       "accuracy": 0.25,
       "cc": -32.25806451612903,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 2,
        "noul": 2
       },
       "score": 0
      },
      "pl": {
       "accuracy": 0.6410256410256411,
       "cc": 35.48908488681278,
       "cc_by_type": {
        "choice": 77.38693467336684,
        "noul": 16.666666666666675,
        "score": 12.41365332040485
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 12,
        "score": 10
       },
       "score": 35.48908488681278
      },
      "pt": {
       "accuracy": 0.7083333333333334,
       "cc": 52.32917576828053,
       "cc_by_type": {
        "choice": 61.918194640338506,
        "noul": 50,
        "score": 45.0693326645031
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 6
       },
       "score": 52.32917576828053
      },
      "sv": {
       "accuracy": 0.25,
       "cc": 7.519324033911375,
       "cc_by_type": {
        "choice": 6.7357512953367875,
        "noul": -33.333333333333336,
        "score": 49.155554139730675
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 7.519324033911375
      },
      "tr": {
       "accuracy": 0.8181818181818182,
       "cc": 76.85175629288626,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 100,
        "score": 74.033529748224
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 3
       },
       "score": 76.85175629288626
      },
      "uk": {
       "accuracy": 0.4444444444444444,
       "cc": 12.647260678131401,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": -33.333333333333336,
        "score": 35.7912443999856
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 2
       },
       "score": 12.647260678131401
      },
      "zh": {
       "accuracy": 0.5,
       "cc": -13.600416695423213,
       "cc_by_type": {
        "choice": 68.08510638297872,
        "noul": -33.333333333333336,
        "score": -75.55302313591503
       },
       "low_n": true,
       "n": 12,
       "n_by_type": {
        "choice": 4,
        "noul": 6,
        "score": 2
       },
       "score": 0
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.7333333333333333,
       "cc": 65.47689829110645,
       "cc_by_type": {
        "choice": 65.47689829110645
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 65.47689829110645
      },
      "noul": {
       "accuracy": 0.608,
       "cc": 21.599999999999998,
       "cc_by_type": {
        "noul": 21.599999999999998
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 21.599999999999998
      },
      "score": {
       "accuracy": 0.6053333333333333,
       "cc": 40.021737739599615,
       "cc_by_type": {
        "score": 40.021737739599615
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 40.021737739599615
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5,
       "cc": 18.513631745529423,
       "cc_by_type": {
        "choice": 36.39575971731449,
        "noul": 0,
        "score": 19.145135519273783
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 12,
        "noul": 12,
        "score": 6
       },
       "score": 18.513631745529423
      },
      "code_linting": {
       "accuracy": 0.5454545454545454,
       "cc": 11.54303802688237,
       "cc_by_type": {
        "choice": 44.099378881987576,
        "noul": -60,
        "score": 50.52973519865953
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 10
       },
       "score": 11.54303802688237
      },
      "customer_support": {
       "accuracy": 0.38461538461538464,
       "cc": -26.722943576168863,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -66.66666666666667,
        "score": -70.0239031922747
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 6,
        "score": 1
       },
       "score": 0
      },
      "demand_forecasting": {
       "accuracy": 0.5238095238095238,
       "cc": 10.709697254585473,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": 0,
        "score": -15.922856288191634
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 4,
        "score": 7
       },
       "score": 10.709697254585473
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6428571428571429,
       "cc": 38.283456663102605,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": 19.999999999999996,
        "score": 60.42414048111109
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 4,
        "noul": 5,
        "score": 5
       },
       "score": 38.283456663102605
      },
      "feature_extraction": {
       "accuracy": 0.625,
       "cc": 14.896396249774858,
       "cc_by_type": {
        "choice": 53.125,
        "noul": 33.33333333333333,
        "score": -41.76914458400876
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 5
       },
       "score": 14.896396249774858
      },
      "financial_crime": {
       "accuracy": 0.5714285714285714,
       "cc": 38.56767797990075,
       "cc_by_type": {
        "choice": 57.44680851063829,
        "noul": 19.999999999999996,
        "score": 38.25622542906396
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 38.56767797990075
      },
      "gaming": {
       "accuracy": 0.6470588235294118,
       "cc": 72.00065469480536,
       "cc_by_type": {
        "choice": 71.89695550351288,
        "noul": 100,
        "score": 44.10500858090322
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 2,
        "score": 6
       },
       "score": 72.00065469480536
      },
      "insurance_claims": {
       "accuracy": 0.5333333333333333,
       "cc": 21.988315835975936,
       "cc_by_type": {
        "choice": 52.63157894736842,
        "noul": 0,
        "score": 13.333368560559379
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 2,
        "score": 5
       },
       "score": 21.988315835975936
      },
      "knowledge_graphs": {
       "accuracy": 0.3333333333333333,
       "cc": -1.437337178217594,
       "cc_by_type": {
        "choice": 20.353982300884958,
        "noul": -42.85714285714286,
        "score": 18.191149021605124
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 10,
        "noul": 7,
        "score": 4
       },
       "score": 0
      },
      "lead_generation": {
       "accuracy": 0.7777777777777778,
       "cc": 63.372521556256174,
       "cc_by_type": {
        "choice": 89.89898989898991,
        "noul": 19.999999999999996,
        "score": 80.21857476977863
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 13,
        "noul": 10,
        "score": 4
       },
       "score": 63.372521556256174
      },
      "legal_compliance": {
       "accuracy": 0.5333333333333333,
       "cc": 30.51519933101532,
       "cc_by_type": {
        "choice": 100,
        "noul": -50,
        "score": 41.545597993045966
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 8,
        "score": 2
       },
       "score": 30.51519933101532
      },
      "llm_guardrails": {
       "accuracy": 0.6666666666666666,
       "cc": 42.73737839090273,
       "cc_by_type": {
        "choice": 83.60655737704917,
        "noul": 14.28571428571428,
        "score": 30.31986350994473
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 8,
        "noul": 7,
        "score": 9
       },
       "score": 42.73737839090273
      },
      "model_routing": {
       "accuracy": 0.7931034482758621,
       "cc": 65.46757086712965,
       "cc_by_type": {
        "choice": 78.5473084817586,
        "noul": 76.27118644067797,
        "score": 41.58421767895237
       },
       "low_n": false,
       "n": 203,
       "n_by_type": {
        "choice": 99,
        "noul": 59,
        "score": 45
       },
       "score": 65.46757086712965
      },
      "moderation_trust_safety": {
       "accuracy": 0.5833333333333334,
       "cc": 42.13343102692523,
       "cc_by_type": {
        "choice": 61.038961038961034,
        "noul": 33.33333333333333,
        "score": 32.02799870848132
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 6,
        "score": 5
       },
       "score": 42.13343102692523
      },
      "recruiting": {
       "accuracy": 0.7647058823529411,
       "cc": 66.50324086313627,
       "cc_by_type": {
        "choice": 62.382445141065844,
        "noul": 50,
        "score": 87.12727744834298
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 6
       },
       "score": 66.50324086313627
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 32.07478294962468,
       "cc_by_type": {
        "choice": 26.650366748166256,
        "noul": 33.33333333333333,
        "score": 36.24064876737445
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 6
       },
       "score": 32.07478294962468
      },
      "scientific_discovery": {
       "accuracy": 0.47619047619047616,
       "cc": 11.88079162408494,
       "cc_by_type": {
        "choice": 27.18446601941747,
        "noul": -50,
        "score": 58.45790885283735
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 8
       },
       "score": 11.88079162408494
      },
      "search_retrieval": {
       "accuracy": 0.7916666666666666,
       "cc": 66.28711066326191,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 38.86133198978572
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 66.28711066326191
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      72.65514052121753,
      80.28072103879629
     ],
     "I": [
      35.14753958034039,
      46.18595337580061
     ],
     "capability": [
      54.830282728287756,
      62.33343067078398
     ],
     "composite": null,
     "seed": 17
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 7.088097744249485,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 78.10292780844634,
     "capability": 59.55298661936648,
     "intelligence": 41.00304543028662
    },
    "per_type": {
     "choice": {
      "accuracy": 0.7333333333333333,
      "calibration": 73.62161072458319,
      "cc": 65.47689829110645,
      "cc_by_type": {
       "choice": 65.47689829110645
      },
      "cc_tiered": 66.73956146750962,
      "cc_tiered_open": 70.2690514819802,
      "cc_tiered_sealed": 63.21007145303903,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 65.47689829110645
     },
     "noul": {
      "accuracy": 0.608,
      "calibration": 87.2543085179046,
      "cc": 21.599999999999998,
      "cc_by_type": {
       "noul": 21.599999999999998
      },
      "cc_tiered": 17.395431681813676,
      "cc_tiered_open": 16.316770186335404,
      "cc_tiered_sealed": 18.474093177291948,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 21.599999999999998
     },
     "score": {
      "accuracy": 0.6053333333333333,
      "calibration": 73.43286418285125,
      "cc": 40.021737739599615,
      "cc_by_type": {
       "score": 40.021737739599615
      },
      "cc_tiered": 38.87414314153658,
      "cc_tiered_open": 40.94059358864256,
      "cc_tiered_sealed": 36.8076926944306,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 40.021737739599615
     }
    },
    "raw_on_A": {
     "calibration": 73.5325444400666,
     "capability": 59.23878738188296,
     "intelligence": 44.945030323699314
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.009333333333333305,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 39.78217888278049,
    "Emphasis on Accuracy 60:20:20": 33.79718479843831,
    "Emphasis on Cost 20:20:60": 42.527227502822605,
    "Emphasis on Speed 20:60:20": 44.82688708985581,
    "Intelligence only": 27.574543698298026
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "org2ai",
   "axes": {
    "calibration": 86.58998638479137,
    "cost": 60.82893594182271,
    "intelligence": 51.951877017848766,
    "speed": 92.782872129877
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.05465848238633132,
      "ece_score": 89.06830352273374,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 328,
      "n_tvd": 0,
      "score": 89.06830352273374,
      "score_before_time_limit": 89.06830352273374,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.16249185347848963,
      "ece": 0.09805762809316519,
      "n": 375,
      "score": 80.38847438136696,
      "score_before_time_limit": 80.38847438136696,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9813333333333333,
      "interval90_width": 0.6019999999999996,
      "n": 375,
      "nrps": 0.1056056821458286,
      "score": 90.31318125027343,
      "score_before_time_limit": 90.31318125027343,
      "time_limit_failed_share": 0,
      "top_ece": 0.04406534642435141
     }
    },
    "score": 86.58998638479137
   },
   "capability": 69.27093170132007,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 3,086,051 input + 0 output tokens over 1,500 decisions = 2057.36733333 tokens/decision. GPU-time cost USD 0.0202163214871573/1,000; old list-price cost USD 0.06172102/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0202163214871573/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020216321487157324
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Wald 4B v2.1",
   "draw": {
    "drawn_on": "2026-10-10",
    "g_med": 10.073244581339713,
    "id": "v1.6-regular-20261010-a2",
    "label": "Regular draw",
    "measured_on": "2026-10-10",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.6.9",
    "results_sha256": "68ca4a4d59f78f5f4ebbe68f6b93b06bd91c12bb267222e47ae73e3f738251fa",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX PRO 6000 96GB), offline read-only container (--network none); official vllm/vllm-openai:v0.30.0@sha256:8a69ffad... image; author's server wald_serve.native_vision from the model repo, --prompt-format plain (author RUNBOOK's JevBench format), serving.json defaults otherwise (effort none = one pass, noul_decisive_floor 0.81)",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 56.69080677700682,
    "I_sealed": 47.21294725869071,
    "base": 51.951877017848766,
    "excess": -0.5953850630236044,
    "gap": 9.477859518316109,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.9521688095628,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 84.43243243243245,
       "hard": 64.1396665618119,
       "judge": 76.26895187870798,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 66.16425120772948,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 32.9560003137282,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 42.310346490056276,
       "hard": 31.56850185540615,
       "judge": 32.18036106421194,
       "standard": 32.21728301648263
      }
     },
     "sealed|choice": {
      "cc": 55.12375685402749,
      "n": {
       "easy": 72,
       "hard": 260,
       "judge": 141,
       "standard": 127
      },
      "tiers": {
       "easy": 62.45698195849411,
       "hard": 49.543805650214345,
       "judge": 57.07980315559839,
       "standard": 59.6829772570641
      }
     },
     "sealed|noul": {
      "cc": 53.117408906882595,
      "n": {
       "easy": 39,
       "hard": 132,
       "judge": 72,
       "standard": 57
      },
      "tiers": {
       "easy": 48.71794871794872,
       "hard": 50,
       "judge": 55.55555555555556,
       "standard": 57.89473684210527
      }
     },
     "sealed|score": {
      "cc": 33.39767601516204,
      "n": {
       "easy": 33,
       "hard": 115,
       "judge": 75,
       "standard": 77
      },
      "tiers": {
       "easy": 38.95455244599806,
       "hard": 28.60554068871497,
       "judge": 38.85138751678833,
       "standard": 32.02294120019875
      }
     }
    }
   },
   "jevbench_score": 68.94809240435124,
   "key": "wald-4b-v2-1",
   "last_measured_on": "2026-10-10",
   "licence": "Apache-2.0 (weights and server code)",
   "listing": "ranked",
   "measured_in": "v1.6.5",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "org2ai/Wald-4B@ce3286f7efd332530e5dccd8a17d6496e029f714",
   "noul_decisive": {
    "acc_among_decisive": 0.7813333333333333,
    "decisive": 375,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 375
   },
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.58998638479137,
     "cost": 46.287007162302075,
     "intelligence": 51.951877017848766,
     "speed": 93.07746270925374
    },
    "board_release": "v1.6.43",
    "capability": 69.27093170132007,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 29,
    "capability_rank": 50,
    "composite_ci95": {
     "A": [
      45.35289667440805,
      55.4572699256173
     ],
     "B": [
      42.75715999124745,
      53.77505802688884
     ],
     "C": [
      31.495067135005595,
      48.32811751847993
     ]
    },
    "jevbench_score": 54.28713266716134,
    "measured_in": "v1.6.5",
    "method": "v1.6",
    "open_board_rank": 38,
    "rank": 62,
    "ranks": {
     "A": 62,
     "B": 62,
     "C": 63,
     "capability": 84
    },
    "scores": {
     "A": 54.28713266716134,
     "B": 52.0059503801659,
     "C": 40.700232429112376
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.20302085168659686,
     "p50_s_raw": 0.026510425843298435,
     "p95_s_adjusted": 0.24250074867159127,
     "p95_s_raw": 0.046250374335795645
    },
    "views": {
     "Balanced 33:33:33": 49.82850819150803,
     "Emphasis on Accuracy 60:20:20": 47.56124181789482,
     "Emphasis on Cost 20:20:60": 45.197608867054875,
     "Emphasis on Speed 20:60:20": 58.630700497081925,
     "Intelligence only": 44.522489120413574
    }
   },
   "rank": 86,
   "ranked": true,
   "ranks": {
    "A": 86,
    "B": 96,
    "C": 100,
    "capability": 107
   },
   "repo": "https://huggingface.co/org2ai/Wald-4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.94809240435124,
    "B": 64.71382712849632,
    "C": 51.691869666538956
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.20795744713395833,
    "p50_s_raw": 0.02897872356697917,
    "p95_s_adjusted": 0.253360151220113,
    "p95_s_raw": 0.05168007561005651
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7586206896551724,
       "cc": 63.68096433104207,
       "cc_by_type": {
        "choice": 75,
        "noul": 66.66666666666667,
        "score": 49.376226326459516
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 16,
        "noul": 6,
        "score": 7
       },
       "score": 63.68096433104207
      },
      "calibration_gen": {
       "accuracy": 0.6160714285714286,
       "cc": 44.10064431077017,
       "cc_by_type": {
        "choice": 46.42857142857144,
        "noul": 61.53846153846154,
        "score": 24.334899965277536
       },
       "low_n": false,
       "n": 112,
       "n_by_type": {
        "choice": 56,
        "noul": 26,
        "score": 30
       },
       "score": 44.10064431077017
      },
      "composite_gen": {
       "accuracy": 0.6836734693877551,
       "cc": 57.621281961194256,
       "cc_by_type": {
        "choice": 74.07407407407408,
        "noul": 64.28571428571428,
        "score": 34.50405752379443
       },
       "low_n": false,
       "n": 98,
       "n_by_type": {
        "choice": 36,
        "noul": 28,
        "score": 34
       },
       "score": 57.621281961194256
      },
      "fairness": {
       "accuracy": 1,
       "cc": 89.90132796385994,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 69.70398389157982
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 12,
        "noul": 6,
        "score": 9
       },
       "score": 89.90132796385994
      },
      "jaggedness": {
       "accuracy": 0.5459459459459459,
       "cc": 28.653270418657403,
       "cc_by_type": {
        "choice": 47.19777674849467,
        "noul": 12.82051282051282,
        "score": 25.94152168696473
       },
       "low_n": false,
       "n": 185,
       "n_by_type": {
        "choice": 97,
        "noul": 39,
        "score": 49
       },
       "score": 28.653270418657403
      },
      "llm_router": {
       "accuracy": 0.7424242424242424,
       "cc": 68.06655796712668,
       "cc_by_type": {
        "choice": 56.33999071063632,
        "noul": 83.56164383561644,
        "score": 64.29803935512727
       },
       "low_n": false,
       "n": 264,
       "n_by_type": {
        "choice": 131,
        "noul": 73,
        "score": 60
       },
       "score": 68.06655796712668
      },
      "long_state": {
       "accuracy": 0.3793103448275862,
       "cc": 4.748724836384873,
       "cc_by_type": {
        "choice": 40.476190476190474,
        "noul": -27.27272727272727,
        "score": 1.0427113056914128
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 11,
        "score": 7
       },
       "score": 4.748724836384873
      },
      "ml_usecase": {
       "accuracy": 0.5161290322580645,
       "cc": 22.614498009598538,
       "cc_by_type": {
        "choice": 35.38461538461538,
        "noul": 50,
        "score": -17.541121355819755
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 15,
        "noul": 8,
        "score": 8
       },
       "score": 22.614498009598538
      },
      "multilingual": {
       "accuracy": 0.6412698412698413,
       "cc": 46.51177232484513,
       "cc_by_type": {
        "choice": 55.217196596506945,
        "noul": 58.62068965517242,
        "score": 25.69743072285603
       },
       "low_n": false,
       "n": 315,
       "n_by_type": {
        "choice": 146,
        "noul": 87,
        "score": 82
       },
       "score": 46.51177232484513
      },
      "negation_pair_gen": {
       "accuracy": 0.6964285714285714,
       "cc": 33.02311941019134,
       "cc_by_type": {
        "choice": 61.904761904761905,
        "noul": 50,
        "score": -12.835403674187873
       },
       "low_n": false,
       "n": 56,
       "n_by_type": {
        "choice": 28,
        "noul": 20,
        "score": 8
       },
       "score": 33.02311941019134
      },
      "option_order": {
       "accuracy": 0.8245614035087719,
       "cc": 77.80244173140953,
       "cc_by_type": {
        "choice": 77.80244173140953
       },
       "low_n": false,
       "n": 57,
       "n_by_type": {
        "choice": 57
       },
       "score": 77.80244173140953
      },
      "perturb_pair": {
       "accuracy": 0.8666666666666667,
       "cc": 84.22071661549569,
       "cc_by_type": {
        "choice": 76.4089121887287,
        "noul": 100,
        "score": 76.25323765775835
       },
       "low_n": false,
       "n": 60,
       "n_by_type": {
        "choice": 34,
        "noul": 14,
        "score": 12
       },
       "score": 84.22071661549569
      },
      "usecase": {
       "accuracy": 0.6033755274261603,
       "cc": 41.10751808936778,
       "cc_by_type": {
        "choice": 57.217190210059755,
        "noul": 43.85964912280702,
        "score": 22.24571493523657
       },
       "low_n": false,
       "n": 237,
       "n_by_type": {
        "choice": 111,
        "noul": 57,
        "score": 69
       },
       "score": 41.10751808936778
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.8181818181818182,
       "cc": 42.61103847074362,
       "cc_by_type": {
        "choice": 73.33333333333334,
        "noul": 50,
        "score": 4.49978207889753
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 2
       },
       "score": 42.61103847074362
      },
      "cs": {
       "accuracy": 0.625,
       "cc": 61.864406779661024,
       "cc_by_type": {
        "choice": 23.72881355932204,
        "noul": 100
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 5,
        "noul": 3
       },
       "score": 61.864406779661024
      },
      "da": {
       "accuracy": 0.5416666666666666,
       "cc": 42.74539505383657,
       "cc_by_type": {
        "choice": 28.994082840236683,
        "noul": 60.00000000000001,
        "score": 39.24210232127302
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 8
       },
       "score": 42.74539505383657
      },
      "de": {
       "accuracy": 0.7333333333333333,
       "cc": 55.337376093377294,
       "cc_by_type": {
        "choice": 78.27398913699457,
        "noul": 55.55555555555556,
        "score": 32.18258358758174
       },
       "low_n": false,
       "n": 45,
       "n_by_type": {
        "choice": 24,
        "noul": 9,
        "score": 12
       },
       "score": 55.337376093377294
      },
      "el": {
       "accuracy": 0.5555555555555556,
       "cc": 11.934924140935678,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": 60.00000000000001,
        "score": -63.954263721771284
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 2
       },
       "score": 11.934924140935678
      },
      "en": {
       "accuracy": 0.6657730116175157,
       "cc": 49.96242193822483,
       "cc_by_type": {
        "choice": 59.169938306225426,
        "noul": 53.38345864661653,
        "score": 37.33386886183252
       },
       "low_n": false,
       "n": 1119,
       "n_by_type": {
        "choice": 577,
        "noul": 266,
        "score": 276
       },
       "score": 49.96242193822483
      },
      "es": {
       "accuracy": 0.8571428571428571,
       "cc": 71.78869471940175,
       "cc_by_type": {
        "choice": 73.81818181818181,
        "noul": 100,
        "score": 41.54790234002343
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 10,
        "noul": 11,
        "score": 7
       },
       "score": 71.78869471940175
      },
      "fi": {
       "accuracy": 0.7777777777777778,
       "cc": 78.87323943661971,
       "cc_by_type": {
        "choice": 57.74647887323943,
        "noul": 100
       },
       "low_n": true,
       "n": 9,
       "n_by_type": {
        "choice": 6,
        "noul": 3
       },
       "score": 78.87323943661971
      },
      "fr": {
       "accuracy": 0.7333333333333333,
       "cc": 53.603908086020034,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 50,
        "score": 36.19959449783442
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 6
       },
       "score": 53.603908086020034
      },
      "hi": {
       "accuracy": 0.43333333333333335,
       "cc": 25.416831317917744,
       "cc_by_type": {
        "choice": 13.043478260869566,
        "noul": 60.00000000000001,
        "score": 3.2070156928836524
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 11,
        "noul": 10,
        "score": 9
       },
       "score": 25.416831317917744
      },
      "id": {
       "accuracy": 0.6666666666666666,
       "cc": 33.87096774193548,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 0
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 4,
        "noul": 2
       },
       "score": 33.87096774193548
      },
      "it": {
       "accuracy": 0.7647058823529411,
       "cc": 54.2758437211133,
       "cc_by_type": {
        "choice": 74.61212976022568,
        "noul": 50,
        "score": 38.21540140311422
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 5,
        "noul": 4,
        "score": 8
       },
       "score": 54.2758437211133
      },
      "ja": {
       "accuracy": 0.65625,
       "cc": 49.17418342165371,
       "cc_by_type": {
        "choice": 51.21951219512196,
        "noul": 71.42857142857142,
        "score": 24.874466641267755
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 49.17418342165371
      },
      "ko": {
       "accuracy": 0.5454545454545454,
       "cc": 55.08222770188039,
       "cc_by_type": {
        "choice": 67.21311475409836,
        "noul": 100,
        "score": -1.9664316484571787
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 4,
        "noul": 2,
        "score": 5
       },
       "score": 55.08222770188039
      },
      "mixed": {
       "accuracy": 0.9090909090909091,
       "cc": 82.54649269370202,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 47.63947808110607
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 1
       },
       "score": 82.54649269370202
      },
      "nl": {
       "accuracy": 0.38461538461538464,
       "cc": 9.711332583883976,
       "cc_by_type": {
        "choice": -0.2785515320334245,
        "noul": 33.33333333333333,
        "score": -3.920784049647974
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 5,
        "noul": 6,
        "score": 2
       },
       "score": 9.711332583883976
      },
      "no": {
       "accuracy": 0.5714285714285714,
       "cc": 50.81967213114754,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 3
       },
       "score": 50.81967213114754
      },
      "pl": {
       "accuracy": 0.575,
       "cc": 34.47099050276897,
       "cc_by_type": {
        "choice": 76.38828159160472,
        "noul": 19.999999999999996,
        "score": 7.024689916702187
       },
       "low_n": false,
       "n": 40,
       "n_by_type": {
        "choice": 16,
        "noul": 10,
        "score": 14
       },
       "score": 34.47099050276897
      },
      "pt": {
       "accuracy": 0.7222222222222222,
       "cc": 74.79648342566703,
       "cc_by_type": {
        "choice": 63.81909547738693,
        "noul": 100,
        "score": 60.570354799614194
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 6,
        "score": 5
       },
       "score": 74.79648342566703
      },
      "sv": {
       "accuracy": 0.375,
       "cc": -8.847329747252749,
       "cc_by_type": {
        "choice": -33.333333333333336,
        "noul": 50,
        "score": -43.20865590842491
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 3,
        "noul": 4,
        "score": 1
       },
       "score": 0
      },
      "tr": {
       "accuracy": 0.7272727272727273,
       "cc": 39.204683185393755,
       "cc_by_type": {
        "choice": 73.6842105263158,
        "noul": 33.33333333333333,
        "score": 10.596505696532155
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 3
       },
       "score": 39.204683185393755
      },
      "uk": {
       "accuracy": 0.5454545454545454,
       "cc": -4.6691240891629535,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": -100,
        "score": 29.47088860207636
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 6,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "zh": {
       "accuracy": 0.75,
       "cc": 52.01465673807612,
       "cc_by_type": {
        "choice": 67.56756756756756,
        "noul": 100,
        "score": -11.523597353339188
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 52.01465673807612
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.6786666666666666,
       "cc": 58.36932112627397,
       "cc_by_type": {
        "choice": 58.36932112627397
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 58.36932112627397
      },
      "noul": {
       "accuracy": 0.7813333333333333,
       "cc": 56.266666666666666,
       "cc_by_type": {
        "noul": 56.266666666666666
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 56.266666666666666
      },
      "score": {
       "accuracy": 0.5066666666666667,
       "cc": 33.21347273527731,
       "cc_by_type": {
        "score": 33.21347273527731
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 33.21347273527731
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.6666666666666666,
       "cc": 47.43548710573032,
       "cc_by_type": {
        "choice": 60.46128500823723,
        "noul": 60.00000000000001,
        "score": 21.845176308953697
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 47.43548710573032
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 41.491312000045305,
       "cc_by_type": {
        "choice": 41.08019639934533,
        "noul": 60.00000000000001,
        "score": 23.393739600790585
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 13,
        "noul": 5,
        "score": 6
       },
       "score": 41.491312000045305
      },
      "customer_support": {
       "accuracy": 0.7,
       "cc": 48.534908440165786,
       "cc_by_type": {
        "choice": 51.61290322580645,
        "noul": 50,
        "score": 43.9918220946909
       },
       "low_n": true,
       "n": 20,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 4
       },
       "score": 48.534908440165786
      },
      "demand_forecasting": {
       "accuracy": 0.38461538461538464,
       "cc": 7.558009820211829,
       "cc_by_type": {
        "choice": 39.7590361445783,
        "noul": 0,
        "score": -17.085006683942815
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 11,
        "noul": 4,
        "score": 11
       },
       "score": 7.558009820211829
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.6111111111111112,
       "cc": 60.831849874775266,
       "cc_by_type": {
        "choice": 100,
        "noul": 60.00000000000001,
        "score": 22.495549624325818
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 7,
        "noul": 5,
        "score": 6
       },
       "score": 60.831849874775266
      },
      "feature_extraction": {
       "accuracy": 0.5517241379310345,
       "cc": 33.80012553149724,
       "cc_by_type": {
        "choice": 48.71794871794871,
        "noul": 19.999999999999996,
        "score": 32.68242787654302
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 15,
        "noul": 5,
        "score": 9
       },
       "score": 33.80012553149724
      },
      "financial_crime": {
       "accuracy": 0.6153846153846154,
       "cc": 42.38729210014478,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 100,
        "score": -7.620732395217833
       },
       "low_n": true,
       "n": 26,
       "n_by_type": {
        "choice": 10,
        "noul": 8,
        "score": 8
       },
       "score": 42.38729210014478
      },
      "gaming": {
       "accuracy": 0.5172413793103449,
       "cc": 29.097480905970304,
       "cc_by_type": {
        "choice": 28.712871287128706,
        "noul": 42.85714285714286,
        "score": 15.722428573639357
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 11,
        "noul": 7,
        "score": 11
       },
       "score": 29.097480905970304
      },
      "insurance_claims": {
       "accuracy": 0.5714285714285714,
       "cc": 59.854855365826985,
       "cc_by_type": {
        "choice": 20.634920634920636,
        "noul": 60.00000000000001,
        "score": 98.92964546256033
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 1
       },
       "score": 59.854855365826985
      },
      "knowledge_graphs": {
       "accuracy": 0.4782608695652174,
       "cc": 34.55736075618144,
       "cc_by_type": {
        "choice": 26.717557251908392,
        "noul": 60.00000000000001,
        "score": 16.954525016635912
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 14,
        "noul": 5,
        "score": 4
       },
       "score": 34.55736075618144
      },
      "lead_generation": {
       "accuracy": 0.48,
       "cc": 32.637731364341626,
       "cc_by_type": {
        "choice": 83.73983739837398,
        "noul": 25,
        "score": -10.826643305349105
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 9
       },
       "score": 32.637731364341626
      },
      "legal_compliance": {
       "accuracy": 0.3076923076923077,
       "cc": -22.12982828671353,
       "cc_by_type": {
        "choice": 34.42622950819673,
        "noul": -100,
        "score": -0.8157143683373169
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 8,
        "noul": 1,
        "score": 4
       },
       "score": 0
      },
      "llm_guardrails": {
       "accuracy": 0.8275862068965517,
       "cc": 70.67750613389478,
       "cc_by_type": {
        "choice": 81.65137614678899,
        "noul": 75,
        "score": 55.381142254895344
       },
       "low_n": true,
       "n": 29,
       "n_by_type": {
        "choice": 14,
        "noul": 8,
        "score": 7
       },
       "score": 70.67750613389478
      },
      "model_routing": {
       "accuracy": 0.7866666666666666,
       "cc": 68.49684059479237,
       "cc_by_type": {
        "choice": 79.09407665505226,
        "noul": 86.36363636363636,
        "score": 40.03280876568852
       },
       "low_n": false,
       "n": 150,
       "n_by_type": {
        "choice": 74,
        "noul": 44,
        "score": 32
       },
       "score": 68.49684059479237
      },
      "moderation_trust_safety": {
       "accuracy": 0.5625,
       "cc": 32.325588967854976,
       "cc_by_type": {
        "choice": 52.63157894736842,
        "noul": 60.00000000000001,
        "score": -15.654812043803478
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 8,
        "noul": 5,
        "score": 3
       },
       "score": 32.325588967854976
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 38.753237007491286,
       "cc_by_type": {
        "choice": 69.84924623115576,
        "noul": 33.33333333333333,
        "score": 13.077131457984759
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 6
       },
       "score": 38.753237007491286
      },
      "risk_assessment": {
       "accuracy": 0.46153846153846156,
       "cc": 18.588938121855957,
       "cc_by_type": {
        "choice": 24.528301886792452,
        "noul": 33.33333333333333,
        "score": -2.0948208545578995
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 7,
        "noul": 3,
        "score": 3
       },
       "score": 18.588938121855957
      },
      "scientific_discovery": {
       "accuracy": 0.375,
       "cc": 6.8446821135291955,
       "cc_by_type": {
        "choice": 28.286852589641434,
        "noul": -14.28571428571429,
        "score": 6.532908036660445
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 7,
        "score": 9
       },
       "score": 6.8446821135291955
      },
      "search_retrieval": {
       "accuracy": 0.8636363636363636,
       "cc": 76.70730935415533,
       "cc_by_type": {
        "choice": 85.68019093078759,
        "noul": 100,
        "score": 44.44173713167841
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 7
       },
       "score": 76.70730935415533
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      80.85810602487076,
      87.75388401396593
     ],
     "I": [
      46.645926045990024,
      56.00010478358418
     ],
     "capability": [
      64.53882410044396,
      71.31542904152454
     ],
     "composite": null,
     "seed": 31
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 10.073244581339713,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 86.58998638479137,
     "capability": 69.27093170132007,
     "intelligence": 51.951877017848766
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6786666666666666,
      "calibration": 89.06830352273374,
      "cc": 58.36932112627397,
      "cc_by_type": {
       "choice": 58.36932112627397
      },
      "cc_tiered": 63.03796283179515,
      "cc_tiered_open": 70.9521688095628,
      "cc_tiered_sealed": 55.12375685402749,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 58.36932112627397
     },
     "noul": {
      "accuracy": 0.7813333333333333,
      "calibration": 80.38847438136696,
      "cc": 56.266666666666666,
      "cc_by_type": {
       "noul": 56.266666666666666
      },
      "cc_tiered": 59.64083005730603,
      "cc_tiered_open": 66.16425120772948,
      "cc_tiered_sealed": 53.117408906882595,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 56.266666666666666
     },
     "score": {
      "accuracy": 0.5066666666666667,
      "calibration": 90.31318125027343,
      "cc": 33.21347273527731,
      "cc_by_type": {
       "score": 33.21347273527731
      },
      "cc_tiered": 33.176838164445115,
      "cc_tiered_open": 32.9560003137282,
      "cc_tiered_sealed": 33.39767601516204,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 33.21347273527731
     }
    },
    "raw_on_A": {
     "calibration": 86.69118099549732,
     "capability": 69.21287139307924,
     "intelligence": 51.734561790661175
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.56336650815807,
    "Emphasis on Accuracy 60:20:20": 58.84904646007949,
    "Emphasis on Cost 20:20:60": 63.01588940750156,
    "Emphasis on Speed 20:60:20": 73.50597511891333,
    "Intelligence only": 51.95187701784877
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Samrat Dutta (samratduttaofficial@gmail.com)",
   "axes": {
    "calibration": 65.7809648808148,
    "cost": 68.67136467637603,
    "intelligence": 5.305239654184986,
    "speed": 77.39270161387154
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.21968406249999994,
      "ece_score": 56.06318750000001,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 56.06318750000001,
      "score_before_time_limit": 56.06318750000001,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.6266666666666667,
      "brier": 0.25745770685333363,
      "ece": 0.15462693333333336,
      "n": 375,
      "score": 68.89041436444444,
      "score_before_time_limit": 69.07461333333333,
      "time_limit_failed_share": 0.0026666666666666666
     },
     "score": {
      "interval90_coverage": 0.904,
      "interval90_width": 0.7419999999999997,
      "n": 375,
      "nrps": 0.18400134444000008,
      "score": 72.389292778,
      "score_before_time_limit": 72.389292778,
      "time_limit_failed_share": 0,
      "top_ece": 0.18410639999999998
     }
    },
    "score": 65.7809648808148
   },
   "capability": 35.5431022674999,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.06778805244371189,
     0.5947334633172188
    ],
    "B": [
     0.04550101039862949,
     0.42548800490265093
    ],
    "C": [
     0.047075036419244376,
     0.41300934952584634
    ]
   },
   "cost": {
    "basis": "ESTIMATE, labelled, and RUN 48'S RATE FOR THIS EXACT ROW AND BASE, CARRIED UNCHANGED: answerdotai/ModernBERT-base IS in the frozen pricing_v15.UNLISTED_BASE_MODELS, so no exact base-model floor applies and the module takes the estimate rate from the row's own registry entry; USD 0.03/M input, 0 output is the rate run 47 (quyet-10-small-en) and run 48 (watersheep on v1.5) already assigned to this base. Output 0 is literal: watersheep/infer.py returns output_tokens 0 and the serving path has no generation step (published 02 Oct rule). x this row's own measured input tokens. Estimated, not charged. No new decision.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.011139300000000001
    },
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.011073574813811781
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 1,
    "n_over_limit_answered_ok": 1,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "WaterSheep (Samrat Dutta, ModernBERT-base encoder with calibrated heads, served by the author's own `watersheep --serve`)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (sha256 901983ae...), 1,500 of 1,500 answered, 0 invalid. RE-MEASUREMENT on v1.6 of the row run 48 measured on the retired v1.5 basis, same weights: samratduttaofficial/WaterSheep @ ebf3705d2d9dc648d5cc6b4cbca7646c6ab48998 (Apache-2.0), package github.com/SamratDuttaOfficial/WaterSheep @ 3ccf02b3 via the author's own install line in an isolated venv (torch 2.14.1+cpu, transformers 5.19.0), served with the author's own command `watersheep --model samratduttaofficial/WaterSheep@ebf3705d... --serve --device cpu`. Their server speaks POST /v1/systemone with Jev's shapes, so the UNCHANGED typesafe adapter applies; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. Their own max_len 512 (watersheep.json) truncates every longer state, the 23 long items included, without refusing; not overridden. RUN ON SANDY'S SHARED CPU (12-core Zen 2, 8 threads) with server and driver inside a loopback-only network namespace and no credential in the environment; load average 8.6-15.4 (median 13.8) logged every 60 s, so the Speed axis is a DECLARED LOWER BOUND. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline. add-requests run 59, benchmarkheaven.com/submit ref 6F1561FC; previously GitHub #179",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "CPU (12-core Zen 2; 8 threads)",
   "intelligence": {
    "I_open": 5.71828897246665,
    "I_sealed": 4.892190335903323,
    "base": 5.305239654184986,
    "excess": -1.7472890058749173,
    "gap": 0.8260986365633265,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 17.15486691739995,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 14.37837837837838,
       "hard": 11.292859389745214,
       "judge": 20.896506262359924,
       "standard": 24.654667224780237
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 0,
       "hard": -82.6086956521739,
       "judge": -50,
       "standard": -33.333333333333336
      }
     },
     "open|score": {
      "cc": 0,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 24.005151515151525,
       "hard": -8.935277777777806,
       "judge": -0.7272881355932403,
       "standard": -31.58629629629628
      }
     },
     "sealed|choice": {
      "cc": 14.67657100770997,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 22.843840070948207,
       "hard": 8.880090497737543,
       "judge": 15.270018621973943,
       "standard": 21.29572607463975
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -7.9999999999999964,
       "hard": -61.165048543689316,
       "judge": -60.526315789473685,
       "standard": -49.29577464788733
      }
     },
     "sealed|score": {
      "cc": 0,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 13.4399290780142,
       "hard": -8.180808823529384,
       "judge": 3.6488023952095605,
       "standard": 1.3909137055837406
      }
     }
    }
   },
   "jevbench_score": 0.19479747591837096,
   "key": "watersheep",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "samratduttaofficial/WaterSheep@ebf3705d2d9dc648d5cc6b4cbca7646c6ab48998",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6906474820143885,
    "decisive_rate": 0.3716577540106952,
    "supported": true,
    "valid": 374
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 65.84236453711111,
     "cost": 68.67136467637603,
     "intelligence": 5.305239654184986,
     "speed": 77.5933742155104
    },
    "board_release": "v1.6.43",
    "capability": 35.57380209564805,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 121,
    "capability_rank": 149,
    "composite_ci95": {
     "A": [
      0.06779535705435,
      0.5949014809956202
     ],
     "B": [
      0.04550364299502931,
      0.4255562687724439
     ],
     "C": [
      0.047080109065520834,
      0.4131260284691807
     ]
    },
    "jevbench_score": 0.19483758736149703,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 171,
    "rank": 215,
    "ranks": {
     "A": 215,
     "B": 215,
     "C": 215,
     "capability": 214
    },
    "scores": {
     "A": 0.19483758736149703,
     "B": 0.13414679955920575,
     "C": 0.13530388011215072
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.9529444739222527,
     "p50_s_raw": 0.4014722369611263,
     "p95_s_adjusted": 1.82639614790678,
     "p95_s_raw": 0.8381980739533901
    },
    "views": {
     "Balanced 33:33:33": 0.15640575110581917,
     "Emphasis on Accuracy 60:20:20": 0.09493747584576766,
     "Emphasis on Cost 20:20:60": 0.2296969891012095,
     "Emphasis on Speed 20:60:20": 0.23287926981068977,
     "Intelligence only": 0.05972759292809448
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 256,
   "ranked": true,
   "ranks": {
    "A": 256,
    "B": 256,
    "C": 256,
    "capability": 256
   },
   "repo": "https://huggingface.co/samratduttaofficial/WaterSheep",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.19479747591837096,
    "B": 0.13413158660410493,
    "C": 0.13527602494331317
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.07510986328125,
    "p50_s_raw": 0.462554931640625,
    "p95_s_adjusted": 1.6954187914729117,
    "p95_s_raw": 0.7727093957364559
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "answerdotai/ModernBERT-base",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      60.09500897475834,
      69.81698087786482
     ],
     "I": [
      3.662660743750301,
      7.923346736956256
     ],
     "capability": [
      32.39438635504768,
      38.32689445842991
     ],
     "composite": {
      "A": [
       0.06778805244371189,
       0.5947334633172188
      ],
      "B": [
       0.04550101039862949,
       0.42548800490265093
      ],
      "C": [
       0.047075036419244376,
       0.41300934952584634
      ]
     },
     "seed": 149
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 65.7809648808148,
     "capability": 35.5431022674999,
     "intelligence": 5.305239654184986
    },
    "per_type": {
     "choice": {
      "accuracy": 0.336,
      "calibration": 56.06318750000001,
      "cc": 15.0838835576859,
      "cc_by_type": {
       "choice": 15.0838835576859
      },
      "cc_tiered": 15.91571896255496,
      "cc_tiered_open": 17.15486691739995,
      "cc_tiered_sealed": 14.67657100770997,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 15.0838835576859
     },
     "noul": {
      "accuracy": 0.256,
      "calibration": 68.89041436444444,
      "cc": -48.8,
      "cc_by_type": {
       "noul": -48.8
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.32266666666666666,
      "calibration": 72.389292778,
      "cc": -1.428035087719337,
      "cc_by_type": {
       "score": -1.428035087719337
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 0
     }
    },
    "raw_on_A": {
     "calibration": 58.9569335572792,
     "capability": 33.53090791683189,
     "intelligence": 8.104882276384586
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.002666666666666706,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.15638155138191312,
    "Emphasis on Accuracy 60:20:20": 0.09493212559285719,
    "Emphasis on Cost 20:20:60": 0.2296656724860263,
    "Emphasis on Speed 20:60:20": 0.23278272559462182,
    "Intelligence only": 0.05972759292809448
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Zaitgeist Labs / Christian Oezbek (c.oezbek@gmail.com, GitHub @coezbek / @zaitgeist-charly)",
   "axes": {
    "calibration": 79.37888213235459,
    "cost": 89.00253166733508,
    "intelligence": 3.759262959387639,
    "speed": 73.82860858123392
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.13684690553745932,
      "ece_score": 72.63061889250814,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 72.63061889250814,
      "score_before_time_limit": 72.63061889250814,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.9865229110512129,
      "brier": 0.2509324734501347,
      "ece": 0.08358625336927226,
      "n": 371,
      "score": 83.28274932614555,
      "score_before_time_limit": 83.28274932614555,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.967479674796748,
      "interval90_width": 0.8308491418247516,
      "n": 369,
      "nrps": 0.16220543914182478,
      "score": 82.22327817841011,
      "score_before_time_limit": 82.22327817841011,
      "time_limit_failed_share": 0,
      "top_ece": 0.09666449864498641
     }
    },
    "score": 79.37888213235459
   },
   "capability": 41.569072545871116,
   "class": "Jev-compatible decision model",
   "composite_ci95": {
    "A": [
     0.004014997550795856,
     0.37828572053602627
    ],
    "B": [
     0.002573281180911842,
     0.2630825532407063
    ],
    "C": [
     0.0027881927436082337,
     0.2626984170389072
    ]
   },
   "cost": {
    "basis": "ESTIMATE (size-class reference; self-hosted open weights, no public tariff). THE SAME CHOICE THE PUBLISHED quyet-1-0-tiny ROW AND RUN 59'S tacet-sonata ROW CARRY FOR THE IDENTICAL mmBERT-small BACKBONE, carried unchanged and not a new judgement: DeepInfra base-size encoders at USD 0.005 per 1M input, 0 output - deepinfra:thenlper/gte-base, deepinfra:BAAI/bge-base-en-v1.5, deepinfra:intfloat/e5-base-v2 and deepinfra:sentence-transformers/all-mpnet-base-v2 are all (0.005, 0.0) in the frozen 25 Sep snapshot - x this row's own measured input tokens. Estimated, not charged. OUTPUT TOKENS ARE 0 BY CONSTRUCTION, verified in the author's source: watt_flash/serve.py:answer() formats a softmax over the option markers of one encoder pass and the package contains no generation path at all (the published 02 Oct rule for classifier read-outs). THE SERVER REPORTS NO usage BLOCK OF ANY KIND, so the input counts come EXACTLY from the author's own unmodified watt_flash.layout.Layout.pack_request - the very number their own WattFlash.decide_batch returns as len(p) and only serve.py drops - under PREREG-R61-AMENDMENT-1, cross-checked three ways (independent reassembly of their documented layout; identity of their packer's Overflow set with the server's own 422 set), with no cost number at all on any disagreement; the counts sit in cost_estimate and never in usage. PRICING REFERENCE NOTE: jhu-clsp/mmBERT-small still has no explicit market-listing decision (pricing_v15.reference() raises), so an UNLISTED_BASE_MODELS entry would require a published pricing update.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.00229019
    },
    "kind": "mixed",
    "usage_estimated_rows": 1477,
    "usd_per_1000": 0.0023258530805687202
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "watt-flash-0.1 (Zaitgeist Labs, 140M mmBERT-small encoder with an option-marker scorer, one forward pass per request, self-hosted)",
   "endpoint_condition": "measured 2026-10-07 on the live v1.6.0/v1.6.1 pool (inputs/input-selfhosted-1500.jsonl, sha256 901983ae...). Weights zaitlabs/watt-flash-0.1 @ d49a3a880b0201b325f2d2299d4c36ffc8735477 (Apache-2.0, public and ungated), model.safetensors sha256 099e611bd27c9542732b89be47bcace54964c00d9c7161ebccb061648c9780bf = THE SUBMITTER'S OWN DECLARED HASH, loaded from a local snapshot with the author's own --model flag (no network during the run). Code github.com/zaitgeist-charly/zaitlabs-watt-flash-0.1 @ e9863f06230ad874a5a2cfca3af922e45161a4d7 (Apache-2.0, = HEAD of main), installed with the author's own line `uv sync` against their own uv.lock in an isolated venv (torch 2.14.0, transformers 5.17.0, safetensors 0.8.0) and served with their own documented command `python -m watt_flash.serve --port 8942`. No file of theirs was edited. Their server answers POST /v1/systemone with Jev's shapes (noul -> {noul: P(true)}; choice probabilities keyed by the criteria keys with choice = argmax; score probabilities keyed '0'..'n-1'), so the UNCHANGED typesafe adapter applies and no mapping is ours; all eight checks of the unchanged smoke gate passed on synthetic requests before any sealed item was sent. THE AUTHOR'S FREE HOSTED ENDPOINT api.inference.zaitlabs.com WAS NEVER USED - the submission states it may train on the data, so no sealed item reached it. THE LONG ITEMS ARE REFUSED, NOT TRUNCATED, by the author's own code: watt_flash/layout.py raises Overflow past its documented 8,192-token window and serve.py answers that with HTTP 422, which their own commit e9863f06 says is deliberate ('answer an over-long request with 422, which JevBench scores as a refusal, not an outage'); those items are counted wrong and no other item failed. A RESUBMISSION of GitHub #113 via the form, which the author asks to dedupe with it. RUN ON SANDY'S SHARED CPU (12-core Zen 2, AVX2, fp32 - the package itself calls model.float() off-accelerator, 8 threads, nice), server and driver inside a loopback-only network namespace (unshare -rn) with no credential in the environment; load average logged every 60 s, so the Speed axis is a DECLARED LOWER BOUND. Latency axis uses the standard self-hosted x2 + 0.15 s adjustment. Serial, one decision per request, no retry, no batching, offline. add-requests run 61, benchmarkheaven.com/submit ref 0874B655 (bh-submit 4cf0bdfb-b6d7-4777-a8ac-350d29c5ac3f)",
   "endpoint_kind": "cpu",
   "full_coverage": true,
   "gpu": "CPU (12-core Zen 2, AVX2; 8 threads)",
   "intelligence": {
    "I_open": 5.1066210946590145,
    "I_sealed": 2.4119048241162644,
    "base": 3.759262959387639,
    "excess": 0.12132862810450629,
    "gap": 2.69471627054275,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 9.15774230012725,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 26.05405405405406,
       "hard": -1.9188424032714584,
       "judge": 12.986156888595913,
       "standard": 17.120133947258267
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -85.71428571428572,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "open|score": {
      "cc": 6.162120983849794,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 20.743030303030295,
       "hard": -4.7528703703703545,
       "judge": 12.24322033898303,
       "standard": 11.579999999999991
      }
     },
     "sealed|choice": {
      "cc": 1.205164808546094,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 38.80718350454514,
       "hard": -2.3190045248868976,
       "judge": -8.9385474860335,
       "standard": 4.668062569281944
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -92,
       "hard": -100,
       "judge": -100,
       "standard": -100
      }
     },
     "sealed|score": {
      "cc": 6.0305496638027005,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 12.903262411347516,
       "hard": -5.752573529411742,
       "judge": 16.985089820359278,
       "standard": 9.728629441624314
      }
     }
    }
   },
   "jevbench_score": 0.07452931157363736,
   "key": "watt-flash-0.1",
   "last_measured_on": "2026-10-07",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "zaitlabs/watt-flash-0.1@d49a3a880b0201b325f2d2299d4c36ffc8735477",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.6,
    "decisive_rate": 0.013477088948787063,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 79.37888213235459,
     "cost": 89.00253166733508,
     "intelligence": 3.759262959387639,
     "speed": 75.84832436427908
    },
    "board_release": "v1.6.43",
    "capability": 41.569072545871116,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 111,
    "capability_rank": 139,
    "composite_ci95": {
     "A": [
      0.004016900605731708,
      0.3790125753277212
     ],
     "B": [
      0.0025739064184890927,
      0.26336355668719996
     ],
     "C": [
      0.0027895143095359085,
      0.2632031773109175
     ]
    },
    "jevbench_score": 0.07461802002995101,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 179,
    "rank": 223,
    "ranks": {
     "A": 223,
     "B": 223,
     "C": 223,
     "capability": 203
    },
    "scores": {
     "A": 0.07461802002995101,
     "B": 0.049670099364146944,
     "C": 0.05181806946524376
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 1.0144243478775024,
     "p50_s_raw": 0.4322121739387512,
     "p95_s_adjusted": 2.564176402240991,
     "p95_s_raw": 1.2070882011204955
    },
    "views": {
     "Balanced 33:33:33": 0.05839101580851302,
     "Emphasis on Accuracy 60:20:20": 0.03436581390194284,
     "Emphasis on Cost 20:20:60": 0.09032934623751702,
     "Emphasis on Speed 20:60:20": 0.08921814434421181,
     "Intelligence only": 0.02125044886845491
    }
   },
   "provenance": {
    "carried_metadata_from": "Pinned author release; add-requests r61/score/withrows/REGISTRY.json measurement record",
    "kind": "v1.6.1 addendum a11 (scorer output sha256 4d2833926c2d, method option B / O1S)"
   },
   "rank": 265,
   "ranked": true,
   "ranks": {
    "A": 265,
    "B": 265,
    "C": 265,
    "capability": 245
   },
   "repo": "https://huggingface.co/zaitlabs/watt-flash-0.1",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 0.07452931157363736,
    "B": 0.04963863644240823,
    "C": 0.051756466370581504
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.1312192991375922,
    "p50_s_raw": 0.49060964956879616,
    "p95_s_adjusted": 3.660937681049108,
    "p95_s_raw": 1.755468840524554
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "jhu-clsp/mmBERT-small",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      74.23510907541693,
      82.71222317398025
     ],
     "I": [
      1.3821722420456268,
      6.668265912418397
     ],
     "capability": [
      38.3327917105868,
      43.940510516442146
     ],
     "composite": {
      "A": [
       0.004014997550795856,
       0.37828572053602627
      ],
      "B": [
       0.002573281180911842,
       0.2630825532407063
      ],
      "C": [
       0.0027881927436082337,
       0.2626984170389072
      ]
     },
     "seed": 150
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 79.37888213235459,
     "capability": 41.569072545871116,
     "intelligence": 3.759262959387639
    },
    "per_type": {
     "choice": {
      "accuracy": 0.25866666666666666,
      "calibration": 72.63061889250814,
      "cc": 5.194054735087065,
      "cc_by_type": {
       "choice": 5.194054735087065
      },
      "cc_tiered": 5.181453554336672,
      "cc_tiered_open": 9.15774230012725,
      "cc_tiered_sealed": 1.205164808546094,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 5.194054735087065
     },
     "noul": {
      "accuracy": 0.008,
      "calibration": 83.28274932614555,
      "cc": -98.4,
      "cc_by_type": {
       "noul": -98.4
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.368,
      "calibration": 82.22327817841011,
      "cc": 4.764543859649162,
      "cc_by_type": {
       "score": 4.764543859649162
      },
      "cc_tiered": 6.096335323826247,
      "cc_tiered_open": 6.162120983849794,
      "cc_tiered_sealed": 6.0305496638027005,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 4.764543859649162
     }
    },
    "raw_on_A": {
     "calibration": 80.11428356201952,
     "capability": 42.17074394730905,
     "intelligence": 4.227204332598573
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 0.05831859122894133,
    "Emphasis on Accuracy 60:20:20": 0.03435074966868935,
    "Emphasis on Cost 20:20:60": 0.09022534426099546,
    "Emphasis on Speed 20:60:20": 0.08891445399924461,
    "Intelligence only": 0.02125044886845491
   }
  },
  {
   "adapter_id": "images/winnow-12b/spec.json:jevbench.adapters.typesafe.TypeSafeAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Eldan Ring",
   "axes": {
    "calibration": 82.95806123333315,
    "cost": 66.95597594555505,
    "intelligence": 59.53366229166759,
    "speed": 86.16642825724779
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.11234009752278087,
      "ece_score": 77.53198049544383,
      "mean_tvd": null,
      "n": 737,
      "n_ece_hard": 307,
      "n_tvd": 0,
      "score": 77.53198049544383,
      "score_before_time_limit": 77.53198049544383,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.14016172506738545,
      "brier": 0.09737227728287662,
      "ece": 0.07944785238178745,
      "n": 371,
      "score": 84.11042952364252,
      "score_before_time_limit": 84.11042952364252,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9214092140921409,
      "interval90_width": 0.2443541102077688,
      "n": 369,
      "nrps": 0.06941649910523198,
      "score": 87.23177368091314,
      "score_before_time_limit": 87.23177368091314,
      "time_limit_failed_share": 0,
      "top_ece": 0.09297401363825267
     }
    },
    "score": 82.95806123333315
   },
   "capability": 71.24586176250037,
   "class": "jev-rebuild",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small gemma, 12B active parameters; throughput 26861.9166667 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 911,483 input + 0 output tokens over 1,624 decisions = 561.258004926 tokens/decision. GPU-time cost USD 0.0126318647927227/1,000; old list-price cost USD 0.0280629002463054/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.0126318647927227/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.012631864792722661
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Winnow-12B Q8",
   "endpoint_condition": "evaluator-owned Lium GPU pod (RTX6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX6000",
   "intelligence": {
    "I_open": 59.81914414395652,
    "I_sealed": 59.24818043937866,
    "base": 59.53366229166759,
    "excess": -2.0024239378603803,
    "gap": 0.5709637045778635,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 70.47816196296756,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 88.32432432432431,
       "hard": 67.9144385026738,
       "judge": 68.35860250494396,
       "standard": 69.8618668899121
      }
     },
     "open|noul": {
      "cc": 50.49758454106281,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 30.000000000000004,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 58.48168592783921,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 61.310580174513184,
       "hard": 65.34102213505747,
       "judge": 48.44185240673705,
       "standard": 58.40831667171895
      }
     },
     "sealed|choice": {
      "cc": 69.74580750216904,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 80.04582070800384,
       "hard": 62.33031674208144,
       "judge": 76.72253258845438,
       "standard": 68.96169478999877
      }
     },
     "sealed|noul": {
      "cc": 52.512988405651086,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 76,
       "hard": 28.155339805825253,
       "judge": 60.52631578947369,
       "standard": 77.46478873239437
      }
     },
     "sealed|score": {
      "cc": 55.48574541031586,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 72.39586027967081,
       "hard": 51.82211180030672,
       "judge": 58.61411366152696,
       "standard": 49.665402818839944
      }
     }
    }
   },
   "jevbench_score": 72.21241410032238,
   "key": "winnow-12b",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0, including the applicable Gemma 4 base/derivative licence terms",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9153605015673981,
    "decisive_rate": 0.8598382749326146,
    "supported": true,
    "valid": 371
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 82.95806123333315,
     "cost": 56.556023424877196,
     "intelligence": 59.53366229166759,
     "speed": 86.69753831262085
    },
    "board_release": "v1.6.43",
    "capability": 71.24586176250037,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 22,
    "capability_rank": 39,
    "composite_ci95": {
     "A": [
      66.45196246998344,
      70.4618307059656
     ],
     "B": [
      63.55759159512183,
      69.03636277102011
     ],
     "C": [
      53.88529001998626,
      70.4618307059656
     ]
    },
    "jevbench_score": 68.88542163459404,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 13,
    "rank": 30,
    "ranks": {
     "A": 30,
     "B": 36,
     "C": 33,
     "capability": 69
    },
    "scores": {
     "A": 68.88542163459404,
     "B": 66.78718784915615,
     "C": 67.81878724910808
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.38378054890781643,
     "p50_s_raw": 0.1168902744539082,
     "p95_s_adjusted": 0.5573951802216469,
     "p95_s_raw": 0.20369759011082347
    },
    "views": {
     "Balanced 33:33:33": 65.19874575043826,
     "Emphasis on Accuracy 60:20:20": 62.8080780809742,
     "Emphasis on Cost 20:20:60": 61.44293550575936,
     "Emphasis on Speed 20:60:20": 72.37789977811173,
     "Intelligence only": 59.53366229166759
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 53,
   "ranked": true,
   "ranks": {
    "A": 53,
    "B": 60,
    "C": 54,
    "capability": 87
   },
   "repo": "https://huggingface.co/EldanRing/Winnow-12B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 72.21241410032238,
    "B": 69.26228701922889,
    "C": 71.09426395896253
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3973340891301632,
    "p50_s_raw": 0.1236670445650816,
    "p95_s_adjusted": 0.6084170100279152,
    "p95_s_raw": 0.2292085050139576
   },
   "status": {
    "answered_ok": 1477,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-12B-it LoRA fine-tune, merged and exported as Q8_0 GGUF",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      78.28501959511246,
      85.05351182481425
     ],
     "I": [
      54.0544167472914,
      64.20476080861759
     ],
     "capability": [
      66.87367358483415,
      74.02873882700322
     ],
     "composite": null,
     "seed": 109
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 82.95806123333315,
     "capability": 71.24586176250037,
     "intelligence": 59.53366229166759
    },
    "per_type": {
     "choice": {
      "accuracy": 0.768,
      "calibration": 77.53198049544383,
      "cc": 70.33051353220351,
      "cc_by_type": {
       "choice": 70.33051353220351
      },
      "cc_tiered": 70.1119847325683,
      "cc_tiered_open": 70.47816196296756,
      "cc_tiered_sealed": 69.74580750216904,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 70.33051353220351
     },
     "noul": {
      "accuracy": 0.7786666666666666,
      "calibration": 84.11042952364252,
      "cc": 55.73333333333332,
      "cc_by_type": {
       "noul": 55.73333333333332
      },
      "cc_tiered": 51.505286473356946,
      "cc_tiered_open": 50.49758454106281,
      "cc_tiered_sealed": 52.512988405651086,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 55.73333333333332
     },
     "score": {
      "accuracy": 0.704,
      "calibration": 87.23177368091314,
      "cc": 56.525484762477454,
      "cc_by_type": {
       "score": 56.525484762477454
      },
      "cc_tiered": 56.98371566907753,
      "cc_tiered_open": 58.48168592783921,
      "cc_tiered_sealed": 55.48574541031586,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 56.525484762477454
     }
    },
    "raw_on_A": {
     "calibration": 80.53670554364732,
     "capability": 70.27835744009714,
     "intelligence": 60.02000933654696
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 69.22354719421911,
    "Emphasis on Accuracy 60:20:20": 64.99221292980529,
    "Emphasis on Cost 20:20:60": 68.29833425010125,
    "Emphasis on Speed 20:60:20": 75.13289106758555,
    "Intelligence only": 59.53366229166759
   }
  },
  {
   "adapter_id": "run/wity-1-auto.spec.json:wity_reasoning_adapter.WityReasoningAdapter",
   "addendum": null,
   "alt": {
    "axes": {
     "calibration": 90.45687167393245,
     "cost": 41.78292309262718,
     "intelligence": 78.56795761907267,
     "speed": 75.05991709119236
    },
    "label": "Base-model reference price (base undisclosed at the author's request)",
    "note": "base model undisclosed at the author's request; frozen base-model reference price applied to the same measured tokens",
    "usd_per_1000": 0.08721059113300492
   },
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "Wity (alphanimble)",
   "axes": {
    "calibration": 85.93671111258612,
    "cost": 58.8349720563948,
    "intelligence": 67.88616994428031,
    "speed": 68.32012851304185
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09748974871312502,
      "ece_score": 80.502050257375,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 78.35532891717834,
      "score_before_time_limit": 80.502050257375,
      "time_limit_failed_share": 0.02666666666666667
     },
     "noul": {
      "abstention_rate": 0.09333333333333338,
      "brier": 0.06247285161168309,
      "ece": 0.035695432538436114,
      "n": 375,
      "score": 91.62276797908194,
      "score_before_time_limit": 92.86091349231278,
      "time_limit_failed_share": 0.013333333333333334
     },
     "score": {
      "interval90_coverage": 0.9466666666666667,
      "interval90_width": 0.19822222222222222,
      "n": 375,
      "nrps": 0.052696939614440746,
      "score": 87.83203644149812,
      "score_before_time_limit": 90.48630127901592,
      "time_limit_failed_share": 0.029333333333333333,
      "top_ece": 0.06878851740262039
     }
    },
    "score": 85.93671111258612
   },
   "capability": 76.91144052843322,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     66.90698553569433,
     70.19728667076625
    ],
    "B": [
     65.96748802157725,
     70.62571507703021
    ],
    "C": [
     66.90698553569433,
     70.19728667076625
    ]
   },
   "cost": {
    "basis": "Wity's stated API tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.074026736
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.023559582938388626
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 36,
    "n_over_limit_answered_ok": 36,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "wity-1 (Wity, reasoning auto)",
   "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=auto as the operator asked for their main row.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 66.6410150463723,
    "I_sealed": 69.13132484218832,
    "base": 67.88616994428031,
    "excess": -5.063697438254261,
    "gap": -2.4903097958160174,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 81.27842432028991,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 69.80182447310474,
       "judge": 84.17930125247197,
       "standard": 92.46546672247803
      }
     },
     "open|noul": {
      "cc": 54.76328502415459,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 39.13043478260869,
       "judge": 60.00000000000001,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 63.8813357946724,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 81.26135069496807,
       "hard": 66.56206622564854,
       "judge": 62.42171848967287,
       "standard": 52.01929344007156
      }
     },
     "sealed|choice": {
      "cc": 75.45166028902736,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 81.37609932747026,
       "hard": 63.348416289592755,
       "judge": 81.3780260707635,
       "standard": 87.80638009607095
      }
     },
     "sealed|noul": {
      "cc": 69.00817290045845,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 87.99999999999999,
       "hard": 51.45631067961165,
       "judge": 71.05263157894737,
       "standard": 91.54929577464787
      }
     },
     "sealed|score": {
      "cc": 62.93414133707918,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 88.63984239469578,
       "hard": 49.15477768475961,
       "judge": 70.29595017077045,
       "standard": 66.59730486237312
      }
     }
    }
   },
   "jevbench_score": 68.9647088778163,
   "key": "wity-1",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary hosted API",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.9408284023668639,
    "decisive_rate": 0.9135135135135135,
    "supported": true,
    "valid": 370
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": 14,
    "axes": {
     "calibration": 87.9497550095679,
     "cost": 58.8349720563948,
     "intelligence": 70.28143483604077,
     "speed": 71.79000307789984
    },
    "board_release": "v1.6.43",
    "capability": 79.11559492280433,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      68.86577835236345,
      71.94201428005243
     ],
     "B": [
      67.99651553112257,
      72.43020923475851
     ],
     "C": [
      68.86577835236345,
      71.94201428005243
     ]
    },
    "jevbench_score": 70.76534010331063,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": 22,
    "ranks": {
     "A": 22,
     "B": 17,
     "C": 21,
     "capability": 28
    },
    "scores": {
     "A": 70.76534010331063,
     "B": 70.66802669351945,
     "C": 70.76534010331063
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 1.5746641159057617,
     "p50_s_raw": 1.5746641159057617,
     "p95_s_adjusted": 4.205443101748823,
     "p95_s_raw": 4.205443101748823
    },
    "views": {
     "Balanced 33:33:33": 66.4382396447495,
     "Emphasis on Accuracy 60:20:20": 67.92395209017961,
     "Emphasis on Cost 20:20:60": 63.172705766286356,
     "Emphasis on Speed 20:60:20": 68.48025157142519,
     "Intelligence only": 70.28143483604077
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": 85,
   "ranked": true,
   "ranks": {
    "A": 85,
    "B": 64,
    "C": 62,
    "capability": 53
   },
   "repo": "https://wity.alphanimble.com/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 68.9647088778163,
    "B": 68.74626813567255,
    "C": 68.9647088778163
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.8473002910614014,
    "p50_s_raw": 1.8473002910614014,
    "p95_s_adjusted": 7.969840868189922,
    "p95_s_raw": 7.969840868189922
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Base model: undisclosed (author request)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      81.89162458329712,
      88.19374313243327
     ],
     "I": [
      62.25329303534609,
      72.47551711346684
     ],
     "capability": [
      72.39990539102567,
      79.72804176993532
     ],
     "composite": {
      "A": [
       66.90698553569433,
       70.19728667076625
      ],
      "B": [
       65.96748802157725,
       70.62571507703021
      ],
      "C": [
       66.90698553569433,
       70.19728667076625
      ]
     },
     "seed": 111
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.93671111258612,
     "capability": 76.91144052843322,
     "intelligence": 67.88616994428031
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8146666666666667,
      "calibration": 78.35532891717834,
      "cc": 76.29851368377176,
      "cc_by_type": {
       "choice": 76.29851368377176
      },
      "cc_tiered": 78.36504230465863,
      "cc_tiered_open": 81.27842432028991,
      "cc_tiered_sealed": 75.45166028902736,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 76.29851368377176
     },
     "noul": {
      "accuracy": 0.848,
      "calibration": 91.62276797908194,
      "cc": 69.6,
      "cc_by_type": {
       "noul": 69.6
      },
      "cc_tiered": 61.885728962306516,
      "cc_tiered_open": 54.76328502415459,
      "cc_tiered_sealed": 69.00817290045845,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 69.6
     },
     "score": {
      "accuracy": 0.7866666666666666,
      "calibration": 87.83203644149812,
      "cc": 63.48262267413758,
      "cc_by_type": {
       "score": 63.48262267413758
      },
      "cc_tiered": 63.40773856587579,
      "cc_tiered_open": 63.8813357946724,
      "cc_tiered_sealed": 62.93414133707918,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 63.48262267413758
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.026666666666666616,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.029333333333333322,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.70508264427026,
    "Emphasis on Accuracy 60:20:20": 65.94105907808395,
    "Emphasis on Cost 20:20:60": 62.22187070905237,
    "Emphasis on Speed 20:60:20": 66.10420012709099,
    "Intelligence only": 67.88616994428031
   }
  },
  {
   "adapter_id": "run/wity-1-always.spec.json:wity_reasoning_adapter.WityReasoningAdapter",
   "addendum": null,
   "alt": {
    "axes": {
     "calibration": 91.0421772337923,
     "cost": 41.78292309262718,
     "intelligence": 79.49960112075699,
     "speed": 74.36141604169042
    },
    "label": "Base-model reference price (base undisclosed at the author's request)",
    "note": "base model undisclosed at the author's request; frozen base-model reference price applied to the same measured tokens",
    "usd_per_1000": 0.08721059113300492
   },
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "Wity (alphanimble)",
   "axes": {
    "calibration": 87.14296044619107,
    "cost": 58.8349720563948,
    "intelligence": 67.11413626626955,
    "speed": 68.5105981536613
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.07469390266334129,
      "ece_score": 85.06121946733174,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 82.906335240826,
      "score_before_time_limit": 85.06121946733174,
      "time_limit_failed_share": 0.025333333333333333
     },
     "noul": {
      "abstention_rate": 0.07999999999999996,
      "brier": 0.07909001541945995,
      "ece": 0.05007116234318407,
      "n": 375,
      "score": 88.78595729761167,
      "score_before_time_limit": 89.98576753136318,
      "time_limit_failed_share": 0.013333333333333334
     },
     "score": {
      "interval90_coverage": 0.9493333333333334,
      "interval90_width": 0.21622222222222207,
      "n": 375,
      "nrps": 0.051449317612291365,
      "score": 89.73658880013555,
      "score_before_time_limit": 92.70308760344582,
      "time_limit_failed_share": 0.032,
      "top_ece": 0.047244465159396204
     }
    },
    "score": 87.14296044619107
   },
   "capability": 77.12854835623031,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     66.94020306458522,
     70.08957190157982
    ],
    "B": [
     65.87195241242884,
     70.26842127647677
    ],
    "C": [
     66.94020306458522,
     70.08957190157982
    ]
   },
   "cost": {
    "basis": "Wity's stated API tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.074026736
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.023559582938388626
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 36,
    "n_over_limit_answered_ok": 36,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "wity-1 (Wity, reasoning always)",
   "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=always; labelled configuration variant.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 67.5119499160984,
    "I_sealed": 66.7163226164407,
    "base": 67.11413626626955,
    "excess": -1.7777603427805482,
    "gap": 0.7956272996576956,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 81.71302445902249,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 96.10810810810811,
       "hard": 71.68921044353571,
       "judge": 88.13447593935398,
       "standard": 84.93093344495604
      }
     },
     "open|noul": {
      "cc": 58.719806763285035,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 56.52173913043479,
       "judge": 50,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 62.10301852598769,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 75.97327196189137,
       "hard": 60.98101749360573,
       "judge": 66.87830350526892,
       "standard": 50.24896640387794
      }
     },
     "sealed|choice": {
      "cc": 73.810452177689,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 85.3669351858695,
       "hard": 64.36651583710406,
       "judge": 77.6536312849162,
       "standard": 81.15531469392782
      }
     },
     "sealed|noul": {
      "cc": 66.72801427882574,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 92,
       "hard": 47.57281553398058,
       "judge": 71.05263157894737,
       "standard": 85.91549295774648
      }
     },
     "sealed|score": {
      "cc": 59.6105013928074,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 83.95007438141664,
       "hard": 56.1573234045365,
       "judge": 56.336971033260674,
       "standard": 59.257366414364654
      }
     }
    }
   },
   "jevbench_score": 69.00315679345155,
   "key": "wity-1-always",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary hosted API",
   "listing": "listed",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": "registry ranked=false or no composite",
   "noul_decisive": {
    "acc_among_decisive": 0.9212827988338192,
    "decisive_rate": 0.927027027027027,
    "supported": true,
    "valid": 370
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 89.25002486738025,
     "cost": 58.8349720563948,
     "intelligence": 69.25112551166924,
     "speed": 71.20296182763504
    },
    "board_release": "v1.6.43",
    "capability": 79.25057518952474,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      68.6626707423781,
      71.62549018189077
     ],
     "B": [
      67.67306476984521,
      71.90451506429471
     ],
     "C": [
      68.6626707423781,
      71.62549018189077
     ]
    },
    "jevbench_score": 70.56449594359258,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": null,
    "scores": {
     "A": 70.56449594359258,
     "B": 70.29785155628596,
     "C": 70.56449594359258
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 1.6885605081915855,
     "p50_s_raw": 1.6885605081915855,
     "p95_s_adjusted": 4.489388496428728,
     "p95_s_raw": 4.489388496428728
    },
    "views": {
     "Balanced 33:33:33": 65.96124511263525,
     "Emphasis on Accuracy 60:20:20": 67.23896089186712,
     "Emphasis on Cost 20:20:60": 62.91315123993267,
     "Emphasis on Speed 20:60:20": 67.96251148296399,
     "Intelligence only": 69.25112551166924
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": null,
   "ranked": false,
   "repo": "https://wity.alphanimble.com/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 69.00315679345155,
    "B": 68.61689325174478,
    "C": 69.00315679345155
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 1.9074126482009888,
    "p50_s_raw": 1.9074126482009888,
    "p95_s_adjusted": 7.3874665338545995,
    "p95_s_raw": 7.3874665338545995
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Base model: undisclosed (author request)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      82.60096012448746,
      88.72350025719587
     ],
     "I": [
      61.704668526310996,
      71.10143835839432
     ],
     "capability": [
      72.6594538006522,
      79.50426160694899
     ],
     "composite": {
      "A": [
       66.94020306458522,
       70.08957190157982
      ],
      "B": [
       65.87195241242884,
       70.26842127647677
      ],
      "C": [
       66.94020306458522,
       70.08957190157982
      ]
     },
     "seed": 112
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 87.14296044619107,
     "capability": 77.12854835623031,
     "intelligence": 67.11413626626955
    },
    "per_type": {
     "choice": {
      "accuracy": 0.8093333333333333,
      "calibration": 82.906335240826,
      "cc": 75.61645652359255,
      "cc_by_type": {
       "choice": 75.61645652359255
      },
      "cc_tiered": 77.76173831835575,
      "cc_tiered_open": 81.71302445902249,
      "cc_tiered_sealed": 73.810452177689,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 75.61645652359255
     },
     "noul": {
      "accuracy": 0.8426666666666667,
      "calibration": 88.78595729761167,
      "cc": 68.53333333333333,
      "cc_by_type": {
       "noul": 68.53333333333333
      },
      "cc_tiered": 62.72391052105539,
      "cc_tiered_open": 58.719806763285035,
      "cc_tiered_sealed": 66.72801427882574,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 68.53333333333333
     },
     "score": {
      "accuracy": 0.7813333333333333,
      "calibration": 89.73658880013555,
      "cc": 61.602397935774036,
      "cc_by_type": {
       "score": 61.602397935774036
      },
      "cc_tiered": 60.85675995939755,
      "cc_tiered_open": 62.10301852598769,
      "cc_tiered_sealed": 59.6105013928074,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 61.602397935774036
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.02533333333333332,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.013333333333333308,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.03200000000000003,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 64.52588954983555,
    "Emphasis on Accuracy 60:20:20": 65.5368572234174,
    "Emphasis on Cost 20:20:60": 62.12233224407349,
    "Emphasis on Speed 20:60:20": 66.06282613355656,
    "Intelligence only": 67.11413626626955
   }
  },
  {
   "adapter_id": "run/wity-1-off.spec.json:wity_reasoning_adapter.WityReasoningAdapter",
   "addendum": null,
   "alt": {
    "axes": {
     "calibration": 83.45289319036758,
     "cost": 41.78292309262718,
     "intelligence": 60.40492797149856,
     "speed": 87.66397534612332
    },
    "label": "Base-model reference price (base undisclosed at the author's request)",
    "note": "base model undisclosed at the author's request; frozen base-model reference price applied to the same measured tokens",
    "usd_per_1000": 0.08721059113300492
   },
   "api_exposure_note": "API measurement: the operator's endpoint received the full sealed set (1,200 items) and the public set, without answers. Earlier API-subset answers were reused; only the missing sealed items were sent (v1.6.1).",
   "api_flag": true,
   "api_subset": false,
   "author": "Wity (alphanimble)",
   "axes": {
    "calibration": 81.82569250484788,
    "cost": 58.8349720563948,
    "intelligence": 44.11116118075945,
    "speed": 87.61214901323027
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.10943535174564842,
      "ece_score": 78.11292965087031,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 78.11292965087031,
      "score_before_time_limit": 78.11292965087031,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.20266666666666666,
      "brier": 0.14300007101908263,
      "ece": 0.10507128193494888,
      "n": 375,
      "score": 78.98574361301023,
      "score_before_time_limit": 78.98574361301023,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.9493333333333334,
      "interval90_width": 0.5226666666666664,
      "n": 375,
      "nrps": 0.09960810647445889,
      "score": 88.37840425066312,
      "score_before_time_limit": 88.37840425066312,
      "time_limit_failed_share": 0,
      "top_ece": 0.06641190425613937
     }
    },
    "score": 81.82569250484788
   },
   "capability": 62.968426842803666,
   "class": "decision-api",
   "composite_ci95": {
    "A": [
     32.15286775090439,
     65.37736939946349
    ],
    "B": [
     28.787777128825557,
     61.33770222198112
    ],
    "C": [
     22.32838038257249,
     45.400950971849646
    ]
   },
   "cost": {
    "basis": "Wity's stated API tariff USD 0.042 per 1M input tokens, output free; measured on the v1.6.1 common cost basis (all items except those outside the Jev reference input range) from this run's token usage",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.074026736
    },
    "kind": "tariff",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.023559582938388626
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "wity-1 (Wity, reasoning off)",
   "endpoint_condition": "operator's hosted production API, measured as is; API default is off, our request sets reasoning=off; labelled configuration variant. The first approximately 70 decisions ran alongside two other streams; the remainder ran alone.",
   "endpoint_kind": "api",
   "full_coverage": true,
   "intelligence": {
    "I_open": 41.233518722452544,
    "I_sealed": 46.98880363906636,
    "base": 44.11116118075945,
    "excess": -8.32867255905206,
    "gap": -5.755284916613817,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 60.211735029468755,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 50.92796476879523,
       "judge": 64.40342781806196,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 29.155969634230512,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 85.71428571428572,
       "hard": 21.739130434782616,
       "judge": 10.000000000000009,
       "standard": 44.44444444444444
      }
     },
     "open|score": {
      "cc": 34.332851503658375,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 66.55807900089734,
       "hard": 31.747271308753067,
       "judge": 45.35548485808892,
       "standard": 6.857448113203668
      }
     },
     "sealed|choice": {
      "cc": 60.2170501413345,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 72.06414899120539,
       "hard": 49.60407239819003,
       "judge": 60.89385474860336,
       "standard": 74.50424929178472
      }
     },
     "sealed|noul": {
      "cc": 37.60945396446127,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 63.99999999999999,
       "hard": 20.388349514563096,
       "judge": 42.10526315789473,
       "standard": 52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 43.13990681140334,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 63.76401506785865,
       "hard": 42.31306223924574,
       "judge": 40.34902565317462,
       "standard": 38.66786356483397
      }
     }
    }
   },
   "jevbench_score": 49.18145184761536,
   "key": "wity-1-off",
   "last_measured_on": "2026-10-06",
   "licence": "proprietary hosted API",
   "listing": "listed",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": "registry ranked=false or no composite",
   "noul_decisive": {
    "acc_among_decisive": 0.8762541806020067,
    "decisive_rate": 0.7973333333333333,
    "supported": true,
    "valid": 375
   },
   "open": "no",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 81.82569250484788,
     "cost": 58.8349720563948,
     "intelligence": 44.11116118075945,
     "speed": 89.55912090143326
    },
    "board_release": "v1.6.43",
    "capability": 62.968426842803666,
    "capability_api_board_rank": null,
    "capability_open_board_rank": null,
    "capability_rank": null,
    "composite_ci95": {
     "A": [
      32.27069947350198,
      65.6460656322266
     ],
     "B": [
      28.862486308497225,
      61.52715969788266
     ],
     "C": [
      22.410207967709706,
      45.58754557793514
     ]
    },
    "jevbench_score": 49.37499462500363,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": null,
    "rank": null,
    "ranks": null,
    "scores": {
     "A": 49.37499462500363,
     "B": 45.396938654270464,
     "C": 34.28819071180808
    },
    "speed": {
     "adjustment": "none (API)",
     "n": 124,
     "p50_s_adjusted": 0.28173011541366577,
     "p50_s_raw": 0.28173011541366577,
     "p95_s_adjusted": 0.39287521950900556,
     "p95_s_raw": 0.39287521950900556
    },
    "views": {
     "Balanced 33:33:33": 45.93426190954148,
     "Emphasis on Accuracy 60:20:20": 40.46467602098598,
     "Emphasis on Cost 20:20:60": 45.877374451824245,
     "Emphasis on Speed 20:60:20": 53.189857817529536,
     "Intelligence only": 34.3325026440474
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.6.0 release row",
    "kind": "v1.6.1 full-set re-score (score-v161-3/out-O1S, O1S, hosted API on S u P, not equated)"
   },
   "rank": null,
   "ranked": false,
   "repo": "https://wity.alphanimble.com/",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 49.18145184761536,
    "B": 45.265912980714305,
    "C": 34.153786005288445
   },
   "speed": {
    "adjustment": "none (API)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.3454003408551216,
    "p50_s_raw": 0.3454003408551216,
    "p95_s_adjusted": 0.501721066609025,
    "p95_s_raw": 0.501721066609025
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Base model: undisclosed (author request)",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      75.68787622816288,
      83.1859182390883
     ],
     "I": [
      37.039067859171155,
      49.769681165865016
     ],
     "capability": [
      57.45466681572874,
      65.81540071488247
     ],
     "composite": {
      "A": [
       32.15286775090439,
       65.37736939946349
      ],
      "B": [
       28.787777128825557,
       61.33770222198112
      ],
      "C": [
       22.32838038257249,
       45.400950971849646
      ]
     },
     "seed": 113
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": true,
    "g_med_used": 2.573387642438244,
    "lane": "api",
    "metric_basis": "S u P (full set, API lane, v1.6.1 amendment; not equated)",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 81.82569250484788,
     "capability": 62.968426842803666,
     "intelligence": 44.11116118075945
    },
    "per_type": {
     "choice": {
      "accuracy": 0.6933333333333334,
      "calibration": 78.11292965087031,
      "cc": 60.781713289694295,
      "cc_by_type": {
       "choice": 60.781713289694295
      },
      "cc_tiered": 60.214392585401626,
      "cc_tiered_open": 60.211735029468755,
      "cc_tiered_sealed": 60.2170501413345,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 60.781713289694295
     },
     "noul": {
      "accuracy": 0.6986666666666667,
      "calibration": 78.98574361301023,
      "cc": 39.733333333333334,
      "cc_by_type": {
       "noul": 39.733333333333334
      },
      "cc_tiered": 33.38271179934589,
      "cc_tiered_open": 29.155969634230512,
      "cc_tiered_sealed": 37.60945396446127,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 39.733333333333334
     },
     "score": {
      "accuracy": 0.5706666666666667,
      "calibration": 88.37840425066312,
      "cc": 43.067156612799565,
      "cc_by_type": {
       "score": 43.067156612799565
      },
      "cc_tiered": 38.73637915753086,
      "cc_tiered_open": 34.332851503658375,
      "cc_tiered_sealed": 43.13990681140334,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 43.067156612799565
     }
    },
    "raw_on_A": null,
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 45.71112779287298,
    "Emphasis on Accuracy 60:20:20": 40.360542246694095,
    "Emphasis on Cost 20:20:60": 45.74356490122393,
    "Emphasis on Speed 20:60:20": 52.65413381303676,
    "Intelligence only": 34.3325026440474
   }
  },
  {
   "addendum": null,
   "api_flag": false,
   "api_subset": false,
   "author": "quevirus (Hugging Face)",
   "axes": {
    "calibration": 85.13454850231766,
    "cost": 80.50803800213268,
    "intelligence": 52.14523729900746,
    "speed": 91.64762937997935
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.06366853885427957,
      "ece_score": 87.26629222914409,
      "mean_tvd": null,
      "n": 740,
      "n_ece_hard": 305,
      "n_tvd": 0,
      "score": 87.26629222914409,
      "score_before_time_limit": 87.26629222914409,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0,
      "brier": 0.15445726927323383,
      "ece": 0.10011666645915095,
      "n": 373,
      "score": 79.9766667081698,
      "score_before_time_limit": 79.9766667081698,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.946236559139785,
      "interval90_width": 0.4881272401433691,
      "n": 372,
      "nrps": 0.10731253403237839,
      "score": 88.1606865696391,
      "score_before_time_limit": 88.1606865696391,
      "time_limit_failed_share": 0,
      "top_ece": 0.06473686728741979
     }
    },
    "score": 85.13454850231766
   },
   "capability": 68.63989290066256,
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 674,634 input + 0 output tokens over 1,485 decisions = 454.298989899 tokens/decision. GPU-time cost USD 0.00446408100405121/1,000; old list-price cost USD 0.0136289696969697/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00446408100405121/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1485,
     "rule": "common",
     "usd_per_1000_all_items": 0.01349268
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.004464081004051206
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Xevn-4B",
   "draw": {
    "drawn_on": "2026-10-12",
    "g_med": 9.839526770355711,
    "id": "v1.6-regular-20261012",
    "label": "Second regular draw",
    "measured_on": "2026-10-24",
    "offset": {
     "C": 0,
     "I": 0
    },
    "offset_ci95": null,
    "release": "v1.7.0",
    "results_sha256": "260ea7e54cb5ef1805cef8652006ddc96a10843541807fe8fe159fa9cdcf7bdc",
    "status": "anchor-runs-pending"
   },
   "endpoint_condition": "Evaluator-owned rented Verda GPU VM, one NVIDIA RTX PRO 6000 Blackwell Server Edition 96 GB (97,887 MiB), FIN-03 (Finland, EU), PAYG non-spot, instance type 1RTXPRO6000.30V, USD 2.5991917808219176/h recorded by the runner. Native engine and first-party HTTP shim in a --network none --read-only container; serial requests, read-only code/model/input, blank credential variables and HF offline flags. In-container cut proof before sealed input: only loopback, no default routes, DNS and TCP fail; model inventory, author and shim hashes checked. Measured 24 Oct 2026 09:20-09:26Z, about USD 0.29 rental; measurement container 126 s, run cap 1592 s, complete=true, deadline_hit=false, rc 0; scored loop 81.776480 s over 1,500 requests. Frozen v1.6-regular-20261012 self-hosted input sha256 dfd36720077ef7c41ace367524896507c7f44ce71306fb3e659e8b85069d995b, checked in the container. Raw output sha256 2526d8bfedc67e8a680615052231a6c32fa62aa800b2e31cbe0aaad7afe1001b. Row cleanup and final node shred verified; exact-id VM/volume teardown confirms provider absent and guard closed at 09:26:37Z. GPU rental is not the benchmark's estimated model tariff.",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "intelligence": {
    "I_open": 57.189441526420964,
    "I_sealed": 47.10103307159395,
    "base": 52.14523729900746,
    "excess": 0.2488816844713,
    "gap": 10.088408454827011,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 65.60765951512785,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 80.54054054054055,
       "hard": 62.25228059138094,
       "judge": 72.31377719182598,
       "standard": 54.79280033486815
      }
     },
     "open|noul": {
      "cc": 69.6425120772947,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 65.21739130434783,
       "judge": 60.00000000000001,
       "standard": 77.77777777777777
      }
     },
     "open|score": {
      "cc": 36.31815298684035,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 52.361958198992475,
       "hard": 31.632819097025468,
       "judge": 39.75557897140969,
       "standard": 32.51077918354005
      }
     },
     "sealed|choice": {
      "cc": 60.25280499391855,
      "n": {
       "easy": 70,
       "hard": 247,
       "judge": 146,
       "standard": 137
      },
      "tiers": {
       "easy": 82.84625158831004,
       "hard": 53.16294668671215,
       "judge": 55.92556317335946,
       "standard": 69.62666104197426
      }
     },
     "sealed|noul": {
      "cc": 53.389344572051726,
      "n": {
       "easy": 58,
       "hard": 110,
       "judge": 65,
       "standard": 67
      },
      "tiers": {
       "easy": 44.827586206896555,
       "hard": 54.54545454545454,
       "judge": 41.53846153846155,
       "standard": 73.13432835820895
      }
     },
     "sealed|score": {
      "cc": 27.66094964881158,
      "n": {
       "easy": 35,
       "hard": 123,
       "judge": 67,
       "standard": 75
      },
      "tiers": {
       "easy": 47.45316925731022,
       "hard": 19.995217233285057,
       "judge": 25.622754060555874,
       "standard": 36.15359805799887
      }
     }
    }
   },
   "jevbench_score": 73.72484160351587,
   "key": "xevn-4b",
   "last_measured_on": "2026-10-24",
   "licence": "Apache-2.0 at the measured pin (model card and LICENSE); public, not gated",
   "listing": "ranked",
   "measured_in": "v1.7.0",
   "measurement_date_status": "measured",
   "method": "v1.7",
   "model_pin": "quevirus/Xevn-4B@ba11f0c2d8100a1e8b7789a1174ddee7c24dcab0",
   "new_in": "v1.7.0",
   "noul_decisive": {
    "acc_among_decisive": 0.7935656836461126,
    "decisive": 373,
    "decisive_rate": 1,
    "n": 375,
    "supported": true,
    "valid": 373
   },
   "paid_fast_lane": false,
   "planned_weight_denominator": 1,
   "rank": 39,
   "ranked": true,
   "ranks": {
    "A": 39,
    "B": 69,
    "C": 92,
    "capability": 112
   },
   "repo": "https://huggingface.co/quevirus/Xevn-4B",
   "scores": {
    "A": 73.72484160351587,
    "B": 68.0892756283705,
    "C": 55.68531025339892
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.23218741199993928,
    "p50_s_raw": 0.04109370599996964,
    "p95_s_adjusted": 0.29471238780007525,
    "p95_s_raw": 0.07235619390003763
   },
   "status": {
    "answered_ok": 1485,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "THE AUTHOR'S OWN INFERENCE, run unmodified: quevirus/Xevn-4B at ba11f0c2d8100a1e8b7789a1174ddee7c24dcab0, sole 139-line author file xevn_infer.py (sha256 3c0e434c03e7cc96675afb86b5a9cb5f76f3af07d20f64035329600512f4a267), wrapped by our first-party shim_xevn.py for HTTP, usage and per-item error recovery. Native Xevn('/model', dtype='bfloat16', device='cuda'), eager text forward, use_cache=False, FP32 final-position hidden state and selected tied-head readout. No generation, chat template, quantization or SDPA substitution. Native calibration preserved: choice temperature 1.1, noul and score 1.2, score_lambda=0; when 0.2<P(yes)<0.8, noul snaps to 0.2 below 0.5 and to 0.8 otherwise (tie 0.5 -> 0.8). Score is the native expected ordinal. Torch 2.13.0, Transformers 5.18.0, tokenizers 0.23.2, safetensors 0.8.0, huggingface-hub 1.33.0; optional fla/causal_conv1d/kernels excluded. vLLM 0.30.0 is the image substrate, not the serving backend. Base image vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90; derived image sha256:bebbb6b15cdef220e99bca031ae8b13a0c48a622e4bfcb01afe48ca07da6f0e5. The submitted code was reviewed and cleared before the run, together with evaluator exposure, the device and native inference without an input cap, as preregistered. The runner was reviewed, findings fixed, and the final changes and hashes rechecked before the run. Source review records Hub-capable from_pretrained and conditional hf_hub_download paths; local /model with temps.json, exact inventory/hashes, offline flags and network-none avoid runtime downloads. No author shell/subprocess, eval/exec, pickle or trust_remote_code path in the reviewed serving code. Weights: Apache-2.0, public, ungated; metadata 4,659,865,088 parameters, two BF16 safetensors shards totalling 9,319,828,160 bytes.  FIRST MEASUREMENT of Xevn-4B (our own intake). AUTHOR'S OWN INFERENCE on our rented hardware, one request at a time; first-party HTTP/usage shim preserves the native prompt, probabilities, temperature calibration and noul 0.2/0.8 confidence hardening. NATIVE OUTPUTS: choice argmax and probabilities, noul P(yes), score expected ordinal and probabilities. AUTHOR EXPOSURE: intake records author-reported English/Korean suite figures, not a comparable Decision Index Full or JevBench draw result; no overlap or contamination audit was run by us. The author's reported figures are not independently verified by us. CONTEXT: native, NO input cap, no evaluator-imposed cap, no truncation, splitting or fallback (ruling on file). The 16,384-token long-item boundary is a preregistration/fit classification, not a serving limit. All 1,500 items were ATTEMPTED. NOT ANSWERED, counted wrong: exactly the 15 pre-registered long positions in the pre-registered long-item set failed with HTTP 500 cuda_out_of_memory; native BF16 eager attention cannot hold these approximately 76k-80.5k-token prompts on the 96 GB card. One BF16 attention score tensor alone is approximately 186-215 GB at the 76,309-82,000-token planning/synthetic bounds. The shim frees CUDA memory after each failed item and keeps serving; no author inference change. This is the pre-registered class B complete outcome, with no failure outside the long set. Answered 1,485 of 1,500 items HTTP 200; longest normal item 1,789 native tokens. 10 s RULE: 0 decisions over 10 s; the 15 OOM items already count wrong as failures, and no additional decision is counted wrong by the time rule in the v1.7 share view (frozen v1.6 and v1.7-off views apply no time limit). Maximum 1.070494 s, median 0.041094 s, p95 0.072361 s over all 1,500 requests; OOM items fail in about 1 s. DETERMINISM: first 50 positions run twice, 50/50 bit-identical, 0 argmax flips, maximum probability difference 0.0. Pre-sealed synthetic probes: 16,384 tokens answered HTTP 200; 76,309 and 82,000 tokens attempted and failed HTTP 500, followed each time by healthy service, a successful short request and GPU memory back to about 9.5 GB. ",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns": {
     "family": {
      "calibration": {
       "accuracy": 0.7727272727272727,
       "cc": 72.73849014174947,
       "cc_by_type": {
        "choice": 58.333333333333336,
        "noul": 100,
        "score": 59.882137091915055
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 16,
        "noul": 4,
        "score": 2
       },
       "score": 72.73849014174947
      },
      "calibration_gen": {
       "accuracy": 0.6138613861386139,
       "cc": 44.81310011472409,
       "cc_by_type": {
        "choice": 58.642629904559925,
        "noul": 60.00000000000001,
        "score": 15.796670439612326
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 25,
        "score": 35
       },
       "score": 44.81310011472409
      },
      "composite_gen": {
       "accuracy": 0.5643564356435643,
       "cc": 29.505656511603615,
       "cc_by_type": {
        "choice": 60.975609756097555,
        "noul": 11.111111111111116,
        "score": 16.430248667602175
       },
       "low_n": false,
       "n": 101,
       "n_by_type": {
        "choice": 41,
        "noul": 27,
        "score": 33
       },
       "score": 29.505656511603615
      },
      "fairness": {
       "accuracy": 1,
       "cc": 95.36247478442279,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 86.08742435326837
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 15,
        "noul": 15,
        "score": 18
       },
       "score": 95.36247478442279
      },
      "jaggedness": {
       "accuracy": 0.5111111111111111,
       "cc": 24.650295817093042,
       "cc_by_type": {
        "choice": 42.252165543792124,
        "noul": 35.13513513513513,
        "score": -3.4364132276481163
       },
       "low_n": false,
       "n": 180,
       "n_by_type": {
        "choice": 94,
        "noul": 37,
        "score": 49
       },
       "score": 24.650295817093042
      },
      "llm_router": {
       "accuracy": 0.7891373801916933,
       "cc": 68.7514533359268,
       "cc_by_type": {
        "choice": 67.50902527075812,
        "noul": 87.62886597938144,
        "score": 51.11646875764083
       },
       "low_n": false,
       "n": 313,
       "n_by_type": {
        "choice": 152,
        "noul": 97,
        "score": 64
       },
       "score": 68.7514533359268
      },
      "long_state": {
       "accuracy": 0,
       "cc": -143.01075268817203,
       "cc_by_type": {
        "choice": -29.032258064516128,
        "noul": -100,
        "score": -300
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 10,
        "noul": 2,
        "score": 3
       },
       "score": 0
      },
      "ml_usecase": {
       "accuracy": 0.48148148148148145,
       "cc": 9.847004603183725,
       "cc_by_type": {
        "choice": 25.773195876288657,
        "noul": -25,
        "score": 28.767817933262517
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 8,
        "score": 8
       },
       "score": 9.847004603183725
      },
      "multilingual": {
       "accuracy": 0.6945454545454546,
       "cc": 54.31575643740647,
       "cc_by_type": {
        "choice": 66.7427453537659,
        "noul": 55.55555555555556,
        "score": 40.64896840289794
       },
       "low_n": false,
       "n": 275,
       "n_by_type": {
        "choice": 134,
        "noul": 72,
        "score": 69
       },
       "score": 54.31575643740647
      },
      "negation_pair_gen": {
       "accuracy": 0.5813953488372093,
       "cc": 15.718276984577555,
       "cc_by_type": {
        "choice": 59.99999999999999,
        "noul": 13.33333333333333,
        "score": -26.178502379600666
       },
       "low_n": false,
       "n": 86,
       "n_by_type": {
        "choice": 40,
        "noul": 30,
        "score": 16
       },
       "score": 15.718276984577555
      },
      "option_order": {
       "accuracy": 0.9583333333333334,
       "cc": 94.67376830892145,
       "cc_by_type": {
        "choice": 94.67376830892145
       },
       "low_n": false,
       "n": 48,
       "n_by_type": {
        "choice": 48
       },
       "score": 94.67376830892145
      },
      "perturb_pair": {
       "accuracy": 0.828125,
       "cc": 78.66172310053767,
       "cc_by_type": {
        "choice": 75.97597597597598,
        "noul": 100,
        "score": 60.009193325637035
       },
       "low_n": false,
       "n": 64,
       "n_by_type": {
        "choice": 44,
        "noul": 10,
        "score": 10
       },
       "score": 78.66172310053767
      },
      "usecase": {
       "accuracy": 0.6181818181818182,
       "cc": 52.426873394583446,
       "cc_by_type": {
        "choice": 55.88235294117649,
        "noul": 66.66666666666667,
        "score": 34.73160057590717
       },
       "low_n": false,
       "n": 220,
       "n_by_type": {
        "choice": 104,
        "noul": 48,
        "score": 68
       },
       "score": 52.426873394583446
      }
     },
     "lang": {
      "ar": {
       "accuracy": 0.5,
       "cc": 25.048133307805642,
       "cc_by_type": {
        "choice": 23.809523809523807,
        "noul": 50,
        "score": 1.3348761138931131
       },
       "low_n": true,
       "n": 16,
       "n_by_type": {
        "choice": 7,
        "noul": 4,
        "score": 5
       },
       "score": 25.048133307805642
      },
      "cs": {
       "accuracy": 0.7142857142857143,
       "cc": 78.94736842105263,
       "cc_by_type": {
        "choice": 57.89473684210526,
        "noul": 100
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 6,
        "noul": 1
       },
       "score": 78.94736842105263
      },
      "da": {
       "accuracy": 0.6923076923076923,
       "cc": 51.83174838150152,
       "cc_by_type": {
        "choice": 100,
        "noul": 33.33333333333333,
        "score": 22.161911811171255
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 3,
        "score": 4
       },
       "score": 51.83174838150152
      },
      "de": {
       "accuracy": 0.84375,
       "cc": 71.69130854287623,
       "cc_by_type": {
        "choice": 84.07079646017698,
        "noul": 75,
        "score": 56.00312916845173
       },
       "low_n": false,
       "n": 32,
       "n_by_type": {
        "choice": 16,
        "noul": 8,
        "score": 8
       },
       "score": 71.69130854287623
      },
      "el": {
       "accuracy": 0.7333333333333333,
       "cc": 69.67338519389119,
       "cc_by_type": {
        "choice": 49.720670391061454,
        "noul": 66.66666666666667,
        "score": 92.63281852394545
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 8,
        "noul": 6,
        "score": 1
       },
       "score": 69.67338519389119
      },
      "en": {
       "accuracy": 0.6681146828844483,
       "cc": 49.101428893202375,
       "cc_by_type": {
        "choice": 61.33803590388204,
        "noul": 58.273381294964025,
        "score": 27.69286948076105
       },
       "low_n": false,
       "n": 1151,
       "n_by_type": {
        "choice": 588,
        "noul": 278,
        "score": 285
       },
       "score": 49.101428893202375
      },
      "es": {
       "accuracy": 0.7419354838709677,
       "cc": 62.81669649155109,
       "cc_by_type": {
        "choice": 80.42414355628058,
        "noul": 100,
        "score": 8.0259459183727
       },
       "low_n": false,
       "n": 31,
       "n_by_type": {
        "choice": 13,
        "noul": 8,
        "score": 10
       },
       "score": 62.81669649155109
      },
      "fi": {
       "accuracy": 0.375,
       "cc": 28.814955843993364,
       "cc_by_type": {
        "choice": 1.6393442622950836,
        "noul": 33.33333333333333,
        "score": 51.472189936351675
       },
       "low_n": true,
       "n": 8,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 1
       },
       "score": 28.814955843993364
      },
      "fr": {
       "accuracy": 0.6428571428571429,
       "cc": 37.98131330045231,
       "cc_by_type": {
        "choice": 57.64705882352941,
        "noul": 0,
        "score": 56.2968810778275
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 6,
        "noul": 2,
        "score": 6
       },
       "score": 37.98131330045231
      },
      "hi": {
       "accuracy": 0.7142857142857143,
       "cc": 37.043463968559664,
       "cc_by_type": {
        "choice": 66.4804469273743,
        "noul": 25,
        "score": 19.64994497830469
       },
       "low_n": true,
       "n": 21,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 5
       },
       "score": 37.043463968559664
      },
      "id": {
       "accuracy": 0.75,
       "cc": 77.77777777777777,
       "cc_by_type": {
        "choice": 55.55555555555555,
        "noul": 100
       },
       "low_n": true,
       "n": 4,
       "n_by_type": {
        "choice": 3,
        "noul": 1
       },
       "score": 77.77777777777777
      },
      "it": {
       "accuracy": 0.8666666666666667,
       "cc": 76.72593226780768,
       "cc_by_type": {
        "choice": 85.97038191738113,
        "noul": 100,
        "score": 44.20741488604191
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 3
       },
       "score": 76.72593226780768
      },
      "ja": {
       "accuracy": 0.68,
       "cc": 55.77291194289271,
       "cc_by_type": {
        "choice": 42.81175536139793,
        "noul": 80,
        "score": 44.50698046728021
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 9,
        "noul": 10,
        "score": 6
       },
       "score": 55.77291194289271
      },
      "ko": {
       "accuracy": 0.5,
       "cc": 35.36568109185147,
       "cc_by_type": {
        "choice": 100,
        "noul": 0,
        "score": 6.097043275554414
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 3,
        "noul": 2,
        "score": 5
       },
       "score": 35.36568109185147
      },
      "mixed": {
       "accuracy": 1,
       "cc": 99.4107347631596,
       "cc_by_type": {
        "choice": 100,
        "noul": 100,
        "score": 98.23220428947874
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 3
       },
       "score": 99.4107347631596
      },
      "nl": {
       "accuracy": 0.5,
       "cc": 25.24092775397595,
       "cc_by_type": {
        "choice": 48.05194805194805,
        "noul": -33.333333333333336,
        "score": 61.00416854331312
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 25.24092775397595
      },
      "no": {
       "accuracy": 0.8333333333333334,
       "cc": 75,
       "cc_by_type": {
        "choice": 100,
        "noul": 50
       },
       "low_n": true,
       "n": 6,
       "n_by_type": {
        "choice": 2,
        "noul": 4
       },
       "score": 75
      },
      "pl": {
       "accuracy": 0.6666666666666666,
       "cc": 44.26397834155194,
       "cc_by_type": {
        "choice": 84.76512907321202,
        "noul": 19.999999999999996,
        "score": 28.0268059514438
       },
       "low_n": false,
       "n": 39,
       "n_by_type": {
        "choice": 17,
        "noul": 10,
        "score": 12
       },
       "score": 44.26397834155194
      },
      "pt": {
       "accuracy": 0.75,
       "cc": 72.00491020494218,
       "cc_by_type": {
        "choice": 68.1791396582204,
        "noul": 100,
        "score": 47.835590956606154
       },
       "low_n": true,
       "n": 28,
       "n_by_type": {
        "choice": 12,
        "noul": 5,
        "score": 11
       },
       "score": 72.00491020494218
      },
      "sv": {
       "accuracy": 0.45454545454545453,
       "cc": 28.80327255884877,
       "cc_by_type": {
        "choice": -2.127659574468081,
        "noul": 60.00000000000001,
        "score": 28.537477251014387
       },
       "low_n": true,
       "n": 11,
       "n_by_type": {
        "choice": 5,
        "noul": 5,
        "score": 1
       },
       "score": 28.80327255884877
      },
      "tr": {
       "accuracy": 0.6153846153846154,
       "cc": 35.571600287366806,
       "cc_by_type": {
        "choice": 34.78260869565218,
        "noul": 50,
        "score": 21.93219216644824
       },
       "low_n": true,
       "n": 13,
       "n_by_type": {
        "choice": 6,
        "noul": 4,
        "score": 3
       },
       "score": 35.571600287366806
      },
      "uk": {
       "accuracy": 0.7142857142857143,
       "cc": 71.12345297189192,
       "cc_by_type": {
        "choice": 67.74193548387096,
        "noul": 100,
        "score": 45.62842343180479
       },
       "low_n": true,
       "n": 7,
       "n_by_type": {
        "choice": 4,
        "noul": 1,
        "score": 2
       },
       "score": 71.12345297189192
      },
      "zh": {
       "accuracy": 0.6,
       "cc": 27.727390622429468,
       "cc_by_type": {
        "choice": 49.152542372881356,
        "noul": 33.33333333333333,
        "score": 0.6962961610737173
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 5,
        "noul": 3,
        "score": 2
       },
       "score": 27.727390622429468
      }
     },
     "type": {
      "choice": {
       "accuracy": 0.708,
       "cc": 62.2840084961442,
       "cc_by_type": {
        "choice": 62.2840084961442
       },
       "low_n": false,
       "n": 750,
       "n_by_type": {
        "choice": 750
       },
       "score": 62.2840084961442
      },
      "noul": {
       "accuracy": 0.7893333333333333,
       "cc": 57.86666666666667,
       "cc_by_type": {
        "noul": 57.86666666666667
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "noul": 375
       },
       "score": 57.86666666666667
      },
      "score": {
       "accuracy": 0.49333333333333335,
       "cc": 30.459370980009403,
       "cc_by_type": {
        "score": 30.459370980009403
       },
       "low_n": false,
       "n": 375,
       "n_by_type": {
        "score": 375
       },
       "score": 30.459370980009403
      }
     },
     "usecase": {
      "advertising": {
       "accuracy": 0.5909090909090909,
       "cc": 31.992860243105383,
       "cc_by_type": {
        "choice": 52,
        "noul": 25,
        "score": 18.978580729316143
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 8,
        "noul": 8,
        "score": 6
       },
       "score": 31.992860243105383
      },
      "code_linting": {
       "accuracy": 0.5833333333333334,
       "cc": 33.778547910211756,
       "cc_by_type": {
        "choice": 62.73291925465839,
        "noul": 33.33333333333333,
        "score": 5.269391142643542
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 7,
        "noul": 9,
        "score": 8
       },
       "score": 33.778547910211756
      },
      "customer_support": {
       "accuracy": 0.7142857142857143,
       "cc": 51.26446710355204,
       "cc_by_type": {
        "choice": 51.21951219512195,
        "noul": 100,
        "score": 2.573889115534189
       },
       "low_n": true,
       "n": 14,
       "n_by_type": {
        "choice": 8,
        "noul": 4,
        "score": 2
       },
       "score": 51.26446710355204
      },
      "demand_forecasting": {
       "accuracy": 0.35294117647058826,
       "cc": 40.84074431959357,
       "cc_by_type": {
        "choice": 6.432748538011691,
        "noul": 100,
        "score": 16.08948442076902
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 11,
        "noul": 3,
        "score": 3
       },
       "score": 40.84074431959357
      },
      "ecommerce_marketplaces": {
       "accuracy": 0.7058823529411765,
       "cc": 32.54651784203781,
       "cc_by_type": {
        "choice": 71.0144927536232,
        "noul": -33.333333333333336,
        "score": 59.95839410582355
       },
       "low_n": true,
       "n": 17,
       "n_by_type": {
        "choice": 9,
        "noul": 3,
        "score": 5
       },
       "score": 32.54651784203781
      },
      "feature_extraction": {
       "accuracy": 0.7368421052631579,
       "cc": -11.142280648793653,
       "cc_by_type": {
        "choice": 74.84276729559748,
        "noul": 33.33333333333333,
        "score": -141.60294257531177
       },
       "low_n": true,
       "n": 19,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 3
       },
       "score": 0
      },
      "financial_crime": {
       "accuracy": 0.6333333333333333,
       "cc": 56.96253848985584,
       "cc_by_type": {
        "choice": 80.3921568627451,
        "noul": 71.42857142857142,
        "score": 19.06688717825101
       },
       "low_n": false,
       "n": 30,
       "n_by_type": {
        "choice": 13,
        "noul": 7,
        "score": 10
       },
       "score": 56.96253848985584
      },
      "gaming": {
       "accuracy": 0.2727272727272727,
       "cc": -0.9722016882584134,
       "cc_by_type": {
        "choice": 16.37630662020906,
        "noul": -50,
        "score": 30.707088315015696
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 6
       },
       "score": 0
      },
      "insurance_claims": {
       "accuracy": 0.3333333333333333,
       "cc": 33.296585222788686,
       "cc_by_type": {
        "choice": -4.166666666666665,
        "noul": 60.00000000000001,
        "score": 44.05642233503272
       },
       "low_n": true,
       "n": 15,
       "n_by_type": {
        "choice": 6,
        "noul": 5,
        "score": 4
       },
       "score": 33.296585222788686
      },
      "knowledge_graphs": {
       "accuracy": 0.4090909090909091,
       "cc": 33.718022386517426,
       "cc_by_type": {
        "choice": 10.064239828693788,
        "noul": 66.66666666666667,
        "score": 24.423160664191823
       },
       "low_n": true,
       "n": 22,
       "n_by_type": {
        "choice": 10,
        "noul": 6,
        "score": 6
       },
       "score": 33.718022386517426
      },
      "lead_generation": {
       "accuracy": 0.72,
       "cc": 59.1340259848485,
       "cc_by_type": {
        "choice": 76.74418604651163,
        "noul": 77.77777777777777,
        "score": 22.88011413025609
       },
       "low_n": true,
       "n": 25,
       "n_by_type": {
        "choice": 11,
        "noul": 9,
        "score": 5
       },
       "score": 59.1340259848485
      },
      "legal_compliance": {
       "accuracy": 0.5,
       "cc": 23.608514769162785,
       "cc_by_type": {
        "choice": 35.483870967741936,
        "noul": 33.33333333333333,
        "score": 2.0083400064130896
       },
       "low_n": true,
       "n": 10,
       "n_by_type": {
        "choice": 4,
        "noul": 3,
        "score": 3
       },
       "score": 23.608514769162785
      },
      "llm_guardrails": {
       "accuracy": 0.6296296296296297,
       "cc": 62.99586745712046,
       "cc_by_type": {
        "choice": 42.41842610364683,
        "noul": 100,
        "score": 46.56917626771456
       },
       "low_n": true,
       "n": 27,
       "n_by_type": {
        "choice": 11,
        "noul": 5,
        "score": 11
       },
       "score": 62.99586745712046
      },
      "model_routing": {
       "accuracy": 0.7926267281105991,
       "cc": 67.82357865219124,
       "cc_by_type": {
        "choice": 74.28571428571429,
        "noul": 90.90909090909092,
        "score": 38.27593076176851
       },
       "low_n": false,
       "n": 217,
       "n_by_type": {
        "choice": 101,
        "noul": 66,
        "score": 50
       },
       "score": 67.82357865219124
      },
      "moderation_trust_safety": {
       "accuracy": 0.6666666666666666,
       "cc": 73.52337478888262,
       "cc_by_type": {
        "choice": 76.96737044145875,
        "noul": 100,
        "score": 43.602753925189106
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 11,
        "noul": 2,
        "score": 5
       },
       "score": 73.52337478888262
      },
      "recruiting": {
       "accuracy": 0.6666666666666666,
       "cc": 73.07772917610153,
       "cc_by_type": {
        "choice": 66.57381615598885,
        "noul": 100,
        "score": 52.65937137231573
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 8,
        "noul": 3,
        "score": 7
       },
       "score": 73.07772917610153
      },
      "risk_assessment": {
       "accuracy": 0.5,
       "cc": 22.7863946384018,
       "cc_by_type": {
        "choice": 55.665024630541865,
        "noul": 0,
        "score": 12.694159284663531
       },
       "low_n": true,
       "n": 18,
       "n_by_type": {
        "choice": 9,
        "noul": 4,
        "score": 5
       },
       "score": 22.7863946384018
      },
      "scientific_discovery": {
       "accuracy": 0.6521739130434783,
       "cc": 39.57707246355063,
       "cc_by_type": {
        "choice": 56.52173913043478,
        "noul": 33.33333333333333,
        "score": 28.876144926883775
       },
       "low_n": true,
       "n": 23,
       "n_by_type": {
        "choice": 9,
        "noul": 6,
        "score": 8
       },
       "score": 39.57707246355063
      },
      "search_retrieval": {
       "accuracy": 0.875,
       "cc": 85.60432469497714,
       "cc_by_type": {
        "choice": 89.3238434163701,
        "noul": 100,
        "score": 67.48913066856133
       },
       "low_n": true,
       "n": 24,
       "n_by_type": {
        "choice": 12,
        "noul": 4,
        "score": 8
       },
       "score": 85.60432469497714
      }
     }
    },
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.67264371922077,
      87.00306451828784
     ],
     "I": [
      46.73117169541663,
      56.15596364566082
     ],
     "capability": [
      63.86721895874797,
      71.11242130154535
     ],
     "composite": null,
     "seed": 85
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "full_set_api": false,
    "g_med_used": 9.839526770355711,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 85.13454850231766,
     "capability": 68.63989290066256,
     "intelligence": 52.14523729900746
    },
    "per_type": {
     "choice": {
      "accuracy": 0.708,
      "calibration": 87.26629222914409,
      "cc": 62.2840084961442,
      "cc_by_type": {
       "choice": 62.2840084961442
      },
      "cc_tiered": 62.930232254523204,
      "cc_tiered_open": 65.60765951512785,
      "cc_tiered_sealed": 60.25280499391855,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 62.2840084961442
     },
     "noul": {
      "accuracy": 0.7893333333333333,
      "calibration": 79.9766667081698,
      "cc": 57.86666666666667,
      "cc_by_type": {
       "noul": 57.86666666666667
      },
      "cc_tiered": 61.51592832467321,
      "cc_tiered_open": 69.6425120772947,
      "cc_tiered_sealed": 53.389344572051726,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 57.86666666666667
     },
     "score": {
      "accuracy": 0.49333333333333335,
      "calibration": 88.1606865696391,
      "cc": 30.459370980009403,
      "cc_by_type": {
       "score": 30.459370980009403
      },
      "cc_tiered": 31.989551317825967,
      "cc_tiered_open": 36.31815298684035,
      "cc_tiered_sealed": 27.66094964881158,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 30.459370980009403
     }
    },
    "raw_on_A": {
     "calibration": 85.60181885840427,
     "capability": 69.85120932695071,
     "intelligence": 54.10059979549715
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.013333333333333308,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.005333333333333301,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.008000000000000007,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.57215517957154,
    "Emphasis on Accuracy 60:20:20": 61.83213763209691,
    "Emphasis on Cost 20:20:60": 74.23692927512121,
    "Emphasis on Speed 20:60:20": 77.72133756787575,
    "Intelligence only": 52.14523729900746
   }
  },
  {
   "adapter_id": null,
   "addendum": null,
   "alt": null,
   "api_flag": false,
   "api_subset": false,
   "author": "Juspay",
   "axes": {
    "calibration": 84.8928101186779,
    "cost": 72.13803206119073,
    "intelligence": 57.47797916441928,
    "speed": 90.84255575155115
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.12367230463806947,
      "ece_score": 75.2655390723861,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 73.96093639513141,
      "score_before_time_limit": 75.2655390723861,
      "time_limit_failed_share": 0.017333333333333333
     },
     "noul": {
      "abstention_rate": 0.19733333333333336,
      "brier": 0.08803851800016456,
      "ece": 0.03517045741230692,
      "n": 375,
      "score": 91.9742721600182,
      "score_before_time_limit": 92.96590851753862,
      "time_limit_failed_share": 0.010666666666666666
     },
     "score": {
      "interval90_coverage": 0.9573333333333334,
      "interval90_width": 0.31933333333333314,
      "n": 375,
      "nrps": 0.07179731457956051,
      "score": 88.7432218008841,
      "score_before_time_limit": 90.18620101715864,
      "time_limit_failed_share": 0.016,
      "top_ece": 0.0622393325386334
     }
    },
    "score": 84.8928101186779
   },
   "capability": 71.18539464154858,
   "class": "Jev-compatible decision model",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: moe gemma, 4B active parameters; throughput 34662.5 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 717,195 input + 1,477 output tokens over 1,477 decisions = 486.57549086 tokens/decision. GPU-time cost USD 0.00848657130636103/1,000; old list-price cost USD 0.0440017941773866/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.00848657130636103/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": {
     "n_items": 1477,
     "rule": "common",
     "usd_per_1000_all_items": 0.15323633999999997
    },
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.008486571306361034
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 23,
    "n_over_limit_answered_ok": 23,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "Xor 26B-A4B (Juspay, Gemma-4-26B-A4B, bf16)",
   "endpoint_condition": "evaluator-owned Lium GPU pod, 1x NVIDIA RTX PRO 6000 Blackwell 96GB (the author's card class), the author's own serving bundle (SGLang image pinned by digest + their compatibility server, single GPU via their documented switch), offline (all credentials unset), driven over loopback only by the JevBench client with its loopback guard; label-free inputs; pod destroyed afterwards; HF Jev Decision Index intake, 6-7 Oct 2026",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "RTX PRO 6000",
   "intelligence": {
    "I_open": 57.66122558340351,
    "I_sealed": 57.294732745435056,
    "base": 57.47797916441928,
    "excess": -2.2068948044697927,
    "gap": 0.3664928379684511,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 74.77679085167708,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 100.00000000000001,
       "hard": 73.57659641396666,
       "judge": 76.26895187870797,
       "standard": 62.32733361239012
      }
     },
     "open|noul": {
      "cc": 43.24154589371981,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": 100,
       "hard": 47.82608695652173,
       "judge": 10.000000000000009,
       "standard": 55.55555555555556
      }
     },
     "open|score": {
      "cc": 54.96534000481365,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 50.747368826312524,
       "hard": 60.18508130733947,
       "judge": 63.88049297699079,
       "standard": 33.26211353074683
      }
     },
     "sealed|choice": {
      "cc": 73.61328423614917,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 86.6972138053359,
       "hard": 67.42081447963801,
       "judge": 77.6536312849162,
       "standard": 73.39573839142753
      }
     },
     "sealed|noul": {
      "cc": 45.734855736359904,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": 80,
       "hard": 22.330097087378633,
       "judge": 50,
       "standard": 69.01408450704226
      }
     },
     "sealed|score": {
      "cc": 52.53605826379611,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 66.57630209327414,
       "hard": 46.082817980300675,
       "judge": 62.2590525874285,
       "standard": 43.83792543059939
      }
     }
    }
   },
   "jevbench_score": 74.00842889463031,
   "key": "xor-26b-a4b",
   "last_measured_on": "2026-10-06",
   "licence": "apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.1",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": "juspay/Xor-26B-A4B@8afd4396492084c7c91846ed3791161d54665d34",
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.939799331103679,
    "decisive_rate": 0.8059299191374663,
    "supported": true,
    "valid": 371
   },
   "open": "open weights",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 86.13921620236111,
     "cost": 50.69588844260597,
     "intelligence": 58.673823517178164,
     "speed": 91.03179229063062
    },
    "board_release": "v1.6.43",
    "capability": 72.40651985976965,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 18,
    "capability_rank": 35,
    "composite_ci95": {
     "A": [
      64.63565801053839,
      68.71970276219201
     ],
     "B": [
      61.75049706211547,
      67.5877402852976
     ],
     "C": [
      49.28381430026326,
      68.71970276219201
     ]
    },
    "jevbench_score": 67.3818936950357,
    "measured_in": "v1.6.1",
    "method": "v1.6",
    "open_board_rank": 22,
    "rank": 42,
    "ranks": {
     "A": 42,
     "B": 40,
     "C": 37,
     "capability": 61
    },
    "scores": {
     "A": 67.3818936950357,
     "B": 65.43945552763522,
     "C": 64.43613635369312
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.26713270749896767,
     "p50_s_raw": 0.058566353749483824,
     "p95_s_adjusted": 0.2951846059411764,
     "p95_s_raw": 0.0725923029705882
    },
    "views": {
     "Balanced 33:33:33": 62.82194404983342,
     "Emphasis on Accuracy 60:20:20": 61.0942481872419,
     "Emphasis on Cost 20:20:60": 57.336197993965754,
     "Emphasis on Speed 20:60:20": 71.71094828654465,
     "Intelligence only": 58.67382351717816
    }
   },
   "provenance": {
    "carried_metadata_from": "author model card + config.json",
    "kind": "v1.6.1 addendum a7 (scorer output sha256 9d512dc13d7d, method option B / O1S)"
   },
   "rank": 31,
   "ranked": true,
   "ranks": {
    "A": 31,
    "B": 54,
    "C": 67,
    "capability": 88
   },
   "repo": "https://huggingface.co/juspay/Xor-26B-A4B",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 74.00842889463031,
    "B": 69.98305635423213,
    "C": 67.91749590687218
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.27011170852929356,
    "p50_s_raw": 0.06005585426464677,
    "p95_s_adjusted": 0.30493060462176796,
    "p95_s_raw": 0.07746530231088398
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "google/gemma-4-26B-A4B-it",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      79.92159889892,
      86.07279175908928
     ],
     "I": [
      51.12198637107415,
      61.99576687100331
     ],
     "capability": [
      66.53160497364678,
      73.32780427342016
     ],
     "composite": null,
     "seed": 127
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 84.8928101186779,
     "capability": 71.18539464154858,
     "intelligence": 57.47797916441928
    },
    "per_type": {
     "choice": {
      "accuracy": 0.804,
      "calibration": 73.96093639513141,
      "cc": 74.93439936341332,
      "cc_by_type": {
       "choice": 74.93439936341332
      },
      "cc_tiered": 74.19503754391312,
      "cc_tiered_open": 74.77679085167708,
      "cc_tiered_sealed": 73.61328423614917,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 74.93439936341332
     },
     "noul": {
      "accuracy": 0.7493333333333333,
      "calibration": 91.9742721600182,
      "cc": 49.86666666666666,
      "cc_by_type": {
       "noul": 49.86666666666666
      },
      "cc_tiered": 44.48820081503986,
      "cc_tiered_open": 43.24154589371981,
      "cc_tiered_sealed": 45.734855736359904,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 49.86666666666666
     },
     "score": {
      "accuracy": 0.6613333333333333,
      "calibration": 88.7432218008841,
      "cc": 52.68777151445312,
      "cc_by_type": {
       "score": 52.68777151445312
      },
      "cc_tiered": 53.75069913430488,
      "cc_tiered_open": 54.96534000481365,
      "cc_tiered_sealed": 52.53605826379611,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 52.68777151445312
     }
    },
    "raw_on_A": {
     "calibration": 81.41272621760669,
     "capability": 70.66742916053838,
     "intelligence": 59.922132103470055
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0.01733333333333331,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0.010666666666666713,
     "n": 375
    },
    "score": {
     "invalid_rate": 0.016000000000000014,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 70.97511278726917,
    "Emphasis on Accuracy 60:20:20": 64.88090898362509,
    "Emphasis on Cost 20:20:60": 71.43575198385071,
    "Emphasis on Speed 20:60:20": 77.77930683274342,
    "Intelligence only": 57.477979164419274
   }
  },
  {
   "adapter_id": "images/zerank-2/spec.json:reranker_adapter.RerankerAdapter",
   "addendum": null,
   "api_exposure_note": null,
   "api_flag": false,
   "api_subset": false,
   "author": "ZeroEntropy",
   "axes": {
    "calibration": 83.20040692874471,
    "cost": 60.56419192241651,
    "intelligence": 15.596611891228248,
    "speed": 81.79510880020156
   },
   "calibration": {
    "parts": {
     "choice": {
      "ece_hard": 0.09594830772201698,
      "ece_score": 80.8103384555966,
      "mean_tvd": null,
      "n": 750,
      "n_ece_hard": 320,
      "n_tvd": 0,
      "score": 80.8103384555966,
      "score_before_time_limit": 80.8103384555966,
      "time_limit_failed_share": 0
     },
     "noul": {
      "abstention_rate": 0.8613333333333333,
      "brier": 0.2189278600834461,
      "ece": 0.0929386859803417,
      "n": 375,
      "score": 81.41226280393165,
      "score_before_time_limit": 81.41226280393165,
      "time_limit_failed_share": 0
     },
     "score": {
      "interval90_coverage": 0.984,
      "interval90_width": 0.9126666666666668,
      "n": 375,
      "nrps": 0.14987040001080157,
      "score": 87.3786195267059,
      "score_before_time_limit": 87.3786195267059,
      "time_limit_failed_share": 0,
      "top_ece": 0.05127860472754006
     }
    },
    "score": 83.20040692874471
   },
   "capability": 49.39850940998648,
   "class": "reranker",
   "composite_ci95": null,
   "cost": {
    "basis": "GPU-time estimate from v1.8.0: one monthly-rented RTX PRO 6000 96 GB, EUR 999/month / 730 hours, ECB USD 1.1133/EUR dated 28 Oct 2026, USD 1.5235/hour; average load 70%. Architecture: small qwen, 4B active parameters; throughput 61525 tokens/s (architecture-class calibration, 29 Oct 2026, efficient-engine prefill; capped at 100,000 tokens/s). The row's own scored cost basis: 3,409,751 input + 0 output tokens over 1,624 decisions = 2099.60036946 tokens/decision. GPU-time cost USD 0.020631316234009/1,000; old list-price cost USD 0.0524900092364532/1,000; applied min(old list-price cost, GPU-time cost) = USD 0.020631316234009/1,000. The GPU-time estimate is lower. Estimated, not a bill. Intelligence, Calibration, measured latency and the common cost-item membership are unchanged. Prior prices remain in the archived v1.7 releases.",
    "common_basis": null,
    "composite_interval_note": "Composite intervals withheld for this cost restatement; the archived intervals use the old cost. Intelligence and Calibration intervals are unchanged.",
    "kind": "estimate",
    "usage_estimated_rows": 0,
    "usd_per_1000": 0.020631316234009007
   },
   "decision_limit": {
    "limit_s": 10,
    "n_over_limit": 0,
    "n_over_limit_answered_ok": 0,
    "rule": "latency_s > limit -> failed",
    "speed_basis": "all decisions given (lane items in the raw), measured latency_s",
    "speed_n": 1500,
    "treatment": "share"
   },
   "display": "ZeroEntropy zerank-2",
   "endpoint_condition": "evaluator-owned Lium GPU pod (A6000), offline read-only container",
   "endpoint_kind": "gpu",
   "full_coverage": true,
   "gpu": "A6000",
   "intelligence": {
    "I_open": 15.496935616681888,
    "I_sealed": 15.696288165774607,
    "base": 15.596611891228248,
    "excess": -2.772740191530963,
    "gap": -0.19935254909271904,
    "penalty": 1,
    "per_type_split": {
     "open|choice": {
      "cc": 36.28749068653345,
      "n": {
       "easy": 33,
       "hard": 68,
       "judge": 32,
       "standard": 17
      },
      "tiers": {
       "easy": 49.40540540540541,
       "hard": 30.16671909405474,
       "judge": 32.76203032300594,
       "standard": 47.25826705734617
      }
     },
     "open|noul": {
      "cc": 0,
      "n": {
       "easy": 14,
       "hard": 23,
       "judge": 20,
       "standard": 18
      },
      "tiers": {
       "easy": -100,
       "hard": -82.6086956521739,
       "judge": -70,
       "standard": -77.77777777777779
      }
     },
     "open|score": {
      "cc": 10.203316163512213,
      "n": {
       "easy": 9,
       "hard": 29,
       "judge": 21,
       "standard": 16
      },
      "tiers": {
       "easy": 13.133808997075057,
       "hard": 3.6957598546269232,
       "judge": 27.616853766877792,
       "standard": -4.367124040546999
      }
     },
     "sealed|choice": {
      "cc": 34.99399091747191,
      "n": {
       "easy": 96,
       "hard": 252,
       "judge": 137,
       "standard": 115
      },
      "tiers": {
       "easy": 50.77969107974283,
       "hard": 27.205882352941163,
       "judge": 36.68528864059592,
       "standard": 40.14041138071193
      }
     },
     "sealed|noul": {
      "cc": 0,
      "n": {
       "easy": 50,
       "hard": 103,
       "judge": 76,
       "standard": 71
      },
      "tiers": {
       "easy": -88,
       "hard": -86.40776699029126,
       "judge": -71.05263157894737,
       "standard": -52.112676056338024
      }
     },
     "sealed|score": {
      "cc": 12.094873579851914,
      "n": {
       "easy": 43,
       "hard": 125,
       "judge": 57,
       "standard": 75
      },
      "tiers": {
       "easy": 16.780298372211313,
       "hard": 6.472246249261849,
       "judge": 17.43542452331028,
       "standard": 12.986589429664797
      }
     }
    }
   },
   "jevbench_score": 3.711230315362979,
   "key": "zerank-2",
   "last_measured_on": "2026-10-02",
   "licence": "Apache-2.0",
   "listing": "ranked",
   "measured_in": "v1.6.0",
   "measurement_date_status": "run output completion day (UTC)",
   "method": "v1.7",
   "model_pin": null,
   "not_ranked_because": null,
   "noul_decisive": {
    "acc_among_decisive": 0.8653846153846154,
    "decisive_rate": 0.13866666666666666,
    "supported": true,
    "valid": 375
   },
   "open": "yes",
   "planned_weight_denominator": 1,
   "previous": {
    "api_board_rank": null,
    "axes": {
     "calibration": 83.20040692874471,
     "cost": 48.39770052432463,
     "intelligence": 15.596611891228248,
     "speed": 82.89783918855927
    },
    "board_release": "v1.6.43",
    "capability": 49.39850940998648,
    "capability_api_board_rank": null,
    "capability_open_board_rank": 92,
    "capability_rank": 119,
    "composite_ci95": {
     "A": [
      1.9752898202498443,
      4.84420234615273
     ],
     "B": [
      1.507128562340162,
      3.903609846255529
     ],
     "C": [
      1.3717290418401693,
      3.364029407050507
     ]
    },
    "jevbench_score": 3.349795360318651,
    "measured_in": "v1.6.0",
    "method": "v1.6",
    "open_board_rank": 150,
    "rank": 190,
    "ranks": {
     "A": 190,
     "B": 193,
     "C": 191,
     "capability": 176
    },
    "scores": {
     "A": 3.349795360318651,
     "B": 2.635183024414276,
     "C": 2.3262467779990628
    },
    "speed": {
     "adjustment": "x2 + 0.15 s (assumption, not measured)",
     "n": 124,
     "p50_s_adjusted": 0.4991468541324139,
     "p50_s_raw": 0.17457342706620693,
     "p95_s_adjusted": 1.0279872846323996,
     "p95_s_raw": 0.4389936423161998
    },
    "views": {
     "Balanced 33:33:33": 2.824157802929915,
     "Emphasis on Accuracy 60:20:20": 2.025225537167865,
     "Emphasis on Cost 20:20:60": 3.299126425382793,
     "Emphasis on Speed 20:60:20": 3.7681689882350637,
     "Intelligence only": 1.4218711229563352
    }
   },
   "provenance": {
    "carried_metadata_from": "v1.5.7",
    "kind": "v1.6.0 full re-measure (score-overnight-12/out-O1S, O1S)"
   },
   "rank": 231,
   "ranked": true,
   "ranks": {
    "A": 231,
    "B": 232,
    "C": 231,
    "capability": 213
   },
   "repo": "https://huggingface.co/zeroentropy/zerank-2-reranker",
   "rescored_in": "v1.7.0",
   "scores": {
    "A": 3.711230315362979,
    "B": 2.8789326577959984,
    "C": 2.577243274557624
   },
   "speed": {
    "adjustment": "x2 + 0.15 s (assumption, not measured)",
    "basis": "all decisions given (lane items in the raw), measured latency_s",
    "n": 1500,
    "p50_s_adjusted": 0.5193217188119889,
    "p50_s_raw": 0.18466085940599442,
    "p95_s_adjusted": 1.2736574321053922,
    "p95_s_raw": 0.5618287160526961
   },
   "status": {
    "answered_ok": 1500,
    "missing": 0,
    "rows": 1500,
    "status": "complete"
   },
   "support": {
    "choice": "native",
    "noul": "native",
    "score": "native"
   },
   "underlying": "Qwen3-Reranker-derived 4B cross-encoder",
   "v16": {
    "api_subset": false,
    "api_subset_tag": null,
    "breakdowns_equated": false,
    "ci95": {
     "B": 1000,
     "C": [
      77.23040686190234,
      86.00830164706042
     ],
     "I": [
      12.746131677720898,
      18.039638464222975
     ],
     "capability": [
      46.43325156721317,
      50.75210427056033
     ],
     "composite": null,
     "seed": 110
    },
    "ci95_n": 1200,
    "complete": true,
    "equated": null,
    "equating_flag": null,
    "equating_offset": null,
    "g_med_used": 2.573387642438244,
    "lane": "selfhosted",
    "metric_basis": "S u P",
    "n_items": 1500,
    "n_public": 300,
    "n_sealed_side": 1200,
    "on_S": {
     "calibration": 83.20040692874471,
     "capability": 49.39850940998648,
     "intelligence": 15.596611891228248
    },
    "per_type": {
     "choice": {
      "accuracy": 0.49866666666666665,
      "calibration": 80.8103384555966,
      "cc": 35.8866269431524,
      "cc_by_type": {
       "choice": 35.8866269431524
      },
      "cc_tiered": 35.64074080200268,
      "cc_tiered_open": 36.28749068653345,
      "cc_tiered_sealed": 34.99399091747191,
      "low_n": false,
      "n": 750,
      "n_by_type": {
       "choice": 750
      },
      "score": 35.8866269431524
     },
     "noul": {
      "accuracy": 0.12,
      "calibration": 81.41226280393165,
      "cc": -76,
      "cc_by_type": {
       "noul": -76
      },
      "cc_tiered": 0,
      "cc_tiered_open": 0,
      "cc_tiered_sealed": 0,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "noul": 375
      },
      "score": 0
     },
     "score": {
      "accuracy": 0.34933333333333333,
      "calibration": 87.3786195267059,
      "cc": 11.246326611904923,
      "cc_by_type": {
       "score": 11.246326611904923
      },
      "cc_tiered": 11.149094871682063,
      "cc_tiered_open": 10.203316163512213,
      "cc_tiered_sealed": 12.094873579851914,
      "low_n": false,
      "n": 375,
      "n_by_type": {
       "score": 375
      },
      "score": 11.246326611904923
     }
    },
    "raw_on_A": {
     "calibration": 80.48954444569493,
     "capability": 48.704671856760854,
     "intelligence": 16.919799267826768
    },
    "set": "S+P"
   },
   "validity": {
    "choice": {
     "invalid_rate": 0,
     "n": 750
    },
    "noul": {
     "invalid_rate": 0,
     "n": 375
    },
    "score": {
     "invalid_rate": 0,
     "n": 375
    }
   },
   "views": {
    "Balanced 33:33:33": 3.143715163211372,
    "Emphasis on Accuracy 60:20:20": 2.2005346638360526,
    "Emphasis on Cost 20:20:60": 3.864971397661625,
    "Emphasis on Speed 20:60:20": 4.147384989194521,
    "Intelligence only": 1.517577178695752
   }
  }
 ],
 "tier_weights": {
  "easy": 0.1,
  "hard": 0.4,
  "judge": 0.3,
  "standard": 0.2
 },
 "types": {
  "choice": 0.3333333333333333,
  "noul": 0.3333333333333333,
  "score": 0.3333333333333333
 },
 "v16": {
  "a2_supplement": {
   "counts": {
    "A": 300,
    "P": 300,
    "S": 300,
    "api_input": 600,
    "api_smoke_public": 5,
    "core": 0,
    "selfhosted_input": 600
   },
   "release": "v1.6.0-A2",
   "set_manifest_sha256": {
    "A": "380f76186ec6e09130154579c6ae1db0eebd1cae1b219339ad7429458c177f5d",
    "P": "6b2321f8359d87769a95c8e52135542d95ccdc353197e1e5c96abb9b616ce5ba",
    "S": "380f76186ec6e09130154579c6ae1db0eebd1cae1b219339ad7429458c177f5d"
   },
   "supplement_of": "v1.6.0"
  },
  "capability_formula": "mean(Intelligence, Calibration)",
  "counts": {
   "A": 300,
   "P": 300,
   "S": 1200,
   "api_input": 1500,
   "api_smoke_public": 5,
   "core": 900,
   "selfhosted_input": 1500
  },
  "equating": {
   "equating_flag": null,
   "min_pool": 5,
   "offsets": {
    "C": 2.3428887908654588,
    "I": -2.209917383979878
   },
   "pool_note": "Historical v1.6.0 equating offsets, kept for the record. Hosted APIs that answered the full set (v1.6.1) are not equated.",
   "rule": "median over complete full-coverage self-hosted systems of metric(S u P) - metric(A u P), per metric (Intelligence, Calibration); API value = clip(raw_A + offset)"
  },
  "equating_A2": {
   "G_med_A2_basis_P_vs_A2": 3.824932047119341,
   "diagnostic_main_A_offset_same_systems": {
    "C": 1.9486117388517954,
    "I": -4.381056886943014
   },
   "equating_flag": null,
   "offset_bootstrap_B": 200,
   "offsets": {
    "C": 1.5638258868659562,
    "I": -0.2887166717454388
   },
   "per_system": {
    "hopper": {
     "C_A2": 87.82068552843725,
     "C_A_main": 87.40904942545944,
     "C_S": 90.07902805935959,
     "I_A2": 34.77120638990772,
     "I_A_main": 41.88389851544817,
     "I_S": 34.73970870443837,
     "dC": 2.2583425309223344,
     "dI": -0.03149768546934695
    },
    "kev-0.6b": {
     "C_A2": 68.46161035185293,
     "C_A_main": 65.17717796738081,
     "C_S": 66.26655558235618,
     "I_A2": 5.866380901403119,
     "I_A_main": 9.100409843273326,
     "I_S": 7.598789169415934,
     "dC": -2.1950547694967497,
     "dI": 1.732408268012815
    },
    "kev-4b": {
     "C_A2": 68.65918994957973,
     "C_A_main": 69.97372951951687,
     "C_S": 68.45285936403559,
     "I_A2": 19.558821195537895,
     "I_A_main": 25.023085194237723,
     "I_S": 19.510707720813777,
     "dC": -0.20633058554413708,
     "dI": -0.04811347472411853
    },
    "kev-8b": {
     "C_A2": 61.3782880436933,
     "C_A_main": 67.24332350856244,
     "C_S": 70.7561729087212,
     "I_A2": 26.42642690493986,
     "I_A_main": 34.2874494947909,
     "I_S": 29.26217941280341,
     "dC": 9.377884865027895,
     "dI": 2.8357525078635497
    },
    "localjev-qwen3.5-4b": {
     "C_A2": 76.77849725935185,
     "C_A_main": 81.40355743693019,
     "C_S": 85.84286362718517,
     "I_A2": 22.674863142962646,
     "I_A_main": 28.26930138473988,
     "I_S": 24.145363218466134,
     "dC": 9.064366367833316,
     "dI": 1.4705000755034874
    },
    "malkuth-4b": {
     "C_A2": 83.46678991361165,
     "C_A_main": 85.35554104202156,
     "C_S": 88.43741591513763,
     "I_A2": 34.73305384360792,
     "I_A_main": 38.819192380023466,
     "I_S": 33.840703805987175,
     "dC": 4.970626001525986,
     "dI": -0.8923500376207443
    },
    "metask-jev-4b": {
     "C_A2": 79.1415556283946,
     "C_A_main": 83.18850868556241,
     "C_S": 84.41575352936586,
     "I_A2": 41.051921128504205,
     "I_A_main": 41.86894313160032,
     "I_S": 39.166454612935595,
     "dC": 5.274197900971259,
     "dI": -1.8854665155686092
    },
    "raw-phi-4-mini": {
     "C_A2": 56.153985406303406,
     "C_A_main": 53.16092441580774,
     "C_S": 57.015465878675315,
     "I_A2": 14.189557846849354,
     "I_A_main": 12.927281214813327,
     "I_S": 13.660237978082595,
     "dC": 0.861480472371909,
     "dI": -0.5293198687667591
    },
    "raw-qwen3-1.7b": {
     "C_A2": 11.425912172792259,
     "C_A_main": 11.45520847967844,
     "C_S": 11.516326812547591,
     "I_A2": 4.994829708920566,
     "I_A_main": 6.997731901083261,
     "I_S": 5.124286554530066,
     "dC": 0.09041463975533226,
     "dI": 0.1294568456095
    },
    "raw-qwen3-4b-instruct-2507": {
     "C_A2": 34.31244267053921,
     "C_A_main": 37.75992930947443,
     "C_S": 35.18175191334879,
     "I_A2": 35.69189443401588,
     "I_A_main": 38.58935302861775,
     "I_S": 33.95117742100547,
     "dC": 0.8693092428095781,
     "dI": -1.7407170130104106
    },
    "raw-qwen3-8b": {
     "C_A2": 34.345565335170505,
     "C_A_main": 38.60246474366818,
     "C_S": 33.721548452619835,
     "I_A2": 30.34325924789806,
     "I_A_main": 31.73129272246627,
     "I_S": 26.54912920590792,
     "dC": -0.62401688255067,
     "dI": -3.7941300419901403
    },
    "typecastlm": {
     "C_A2": 76.627802179825,
     "C_A_main": 79.3986887571586,
     "C_S": 83.29951051808085,
     "I_A2": 31.896472425382363,
     "I_A_main": 31.998923946026117,
     "I_S": 29.78900656204624,
     "dC": 6.671708338255854,
     "dI": -2.107465863336124
    }
   },
   "pool": [
    "hopper",
    "kev-0.6b",
    "kev-4b",
    "kev-8b",
    "localjev-qwen3.5-4b",
    "malkuth-4b",
    "metask-jev-4b",
    "raw-phi-4-mini",
    "raw-qwen3-1.7b",
    "raw-qwen3-4b-instruct-2507",
    "raw-qwen3-8b",
    "typecastlm"
   ],
   "pool_n": 12,
   "rule": "median over the A2 pool of metric(S u P, main run) - metric(A2 u P, A2 run), per metric; API A2 value = clip(raw_A2 + offset_A2)",
   "runs_status": {
    "hopper": {
     "answered_ok": 600,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "kev-0.6b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "kev-4b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "kev-8b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "localjev-qwen3.5-4b": {
     "answered_ok": 600,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "malkuth-4b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "metask-jev-4b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "raw-phi-4-mini": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "raw-qwen3-1.7b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "raw-qwen3-4b-instruct-2507": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "raw-qwen3-8b": {
     "answered_ok": 594,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    },
    "typecastlm": {
     "answered_ok": 600,
     "missing": 0,
     "rows": 600,
     "status": "complete"
    }
   }
  },
  "full_set_api_rows": [
   "fastino-gliner-2-5-decide",
   "jev-1.13.0",
   "sage-1.3.0",
   "wity-1",
   "wity-1-always",
   "wity-1-off"
  ],
  "g_med_rule": "median gap over ranked self-hosted systems; API systems use the same systems' P-vs-A gap",
  "item_sets": [
   {
    "answered_by": "self-hosted systems (run offline on our own GPU pods or Sandy) and, from v1.6.1, externally hosted APIs",
    "by_type": {
     "choice": 600,
     "noul": 300,
     "score": 300
    },
    "items": 1200,
    "name": "Sealed release draw",
    "set": "S"
   },
   {
    "answered_by": "every system (inside S since v1.6.1); retired for future draws",
    "by_type": {
     "choice": 150,
     "noul": 75,
     "score": 75
    },
    "items": 300,
    "name": "API subset (part of S; used by the v1.6.0 hosted-API measurements)",
    "set": "A"
   },
   {
    "answered_by": "every system",
    "by_type": {
     "choice": 150,
     "noul": 75,
     "score": 75
    },
    "items": 300,
    "name": "Public set",
    "set": "P"
   }
  ],
  "long_items_note": "23 items of this draw are very long (about 77,000 to 81,000 input tokens). Systems with a shorter context window refuse them; Jev-Omni runs out of GPU memory on them on its listed RTX 6000 (48 GB) recipe; decider-4b v2's server truncates them to about 32,800 tokens and answers; Plumb-4B reads them in full. Each outcome is the system's own recipe and is scored as such.",
  "offset_bootstrap_B": 200,
  "packaging_partial": false,
  "release": "v1.6.0",
  "scorer_sources_sha256": {
   "noul_method_v16.py": "119e59486ab3300b972fbcbadb0fe1dceceb124287607375aa61c66063c607d0",
   "score_v15.py": "0931e05c6410b1b971201834397cb609cdb25f942723c50375632952f6dcc447",
   "score_v15_headlineA.py": "826f2cbcb57007ec8a161e6eb0111182b8a656c0638027c6c78022a858fd8e2c",
   "score_v16.py": "02132271117653e6302c509232a581d0094f0e78eac8b1bfc67596fd5d5097e9"
  },
  "scoring_parameters": {
   "B": 1000,
   "g_med_fixed": null,
   "noul_method": "O1S",
   "only": null,
   "seed": 16,
   "test_allow_partial": false,
   "workers": 10
  },
  "set_manifest_sha256": {
   "A": "19b196a71d58da260cd83069ffcd64bb323e158c24a5fcabeffbb5bee7c1eabb",
   "P": "6b2321f8359d87769a95c8e52135542d95ccdc353197e1e5c96abb9b616ce5ba",
   "S": "f13557a141fba12efc6bf6d39a0dd8e3bf803d4cacd6eab516d4f4f87045539a"
  }
 },
 "views": [
  "Balanced 33:33:33",
  "Emphasis on Accuracy 60:20:20",
  "Emphasis on Cost 20:20:60",
  "Emphasis on Speed 20:60:20",
  "Intelligence only"
 ]
}